{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/usr/local/lib/python3.6/site-packages/h5py/__init__.py:36: FutureWarning: Conversion of the second argument of issubdtype from `float` to `np.floating` is deprecated. In future, it will be treated as `np.float64 == np.dtype(float).type`.\n",
      "  from ._conv import register_converters as _register_converters\n"
     ]
    }
   ],
   "source": [
    "# importing the dependencies\n",
    "import numpy as np # linear algebra\n",
    "import pandas as pd # dataframe\n",
    "import tensorflow as tf # Machine learning\n",
    "from glob import glob # file handling\n",
    "from tqdm import tqdm # progress bar\n",
    "from collections import deque # for simpler implementation of memory\n",
    "\n",
    "import matplotlib.pyplot as plt # visualisation\n",
    "%matplotlib inline"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Some helper functions below\n",
    "\n",
    "Below is the code for some helper functions and is consistent across various models."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "# function to properly return the string of price\n",
    "def format_price(price):\n",
    "    return (\"-$\" if price < 0 else \"$\") + \"{0:.2f}\".format(abs(price))\n",
    "\n",
    "def sigmoid(x):\n",
    "    return 1/(1 + np.exp(-x))\n",
    "\n",
    "# function to get the state\n",
    "def get_state(data, t, n):\n",
    "    d = t - n + 1\n",
    "    if d >= 0:\n",
    "        block = data[d:t+1]\n",
    "    else:\n",
    "        # pad with t0\n",
    "        block = -d*[data[0]] + data[0:t+1].tolist()\n",
    "        \n",
    "    # block = data[d:t + 1] if d >= 0 else -d * [data[0]] + data[0:t + 1] # pad with t0\n",
    "    \n",
    "    # get results\n",
    "    res = []\n",
    "    for i in range(n - 1):\n",
    "        res.append(sigmoid(block[i + 1] - block[i]))\n",
    "    \n",
    "    # return numpy array\n",
    "    return np.array([res])"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Load Data\n",
    "\n",
    "The code below is to load the data and is similar across various models."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [],
   "source": [
    "# load the csv file\n",
    "path_folder = './data_raw/'\n",
    "file_path = glob(path_folder + '*.csv')[0]\n",
    "stock_name = file_path.split('.')[-2].split('/')[-1]\n",
    "data = pd.read_csv(file_path)\n",
    "\n",
    "# constants\n",
    "LOG = False\n",
    "episode_count = 10\n",
    "window_size = 100\n",
    "data = data['Close'].values # what our data is\n",
    "len_data = len(data) - 1 # total information length\n",
    "batch_size = 32 # minibatch size\n",
    "\n",
    "# logs\n",
    "loss_global = []\n",
    "profits_global = []"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {
    "scrolled": false
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "Text(0.5,1,'AMZN- Amazon')"
      ]
     },
     "execution_count": 4,
     "metadata": {},
     "output_type": "execute_result"
    },
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAABI4AAAHmCAYAAAAV9MUJAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADl0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uIDIuMi4yLCBodHRwOi8vbWF0cGxvdGxpYi5vcmcvhp/UCwAAIABJREFUeJzs3XmYnWV9//H3d/Yly2QZkpAEAhJk1SAIWMGlKgKiqHWBuqBF0Z/WWmvdan91aWm1Vq2o1aLyU3ArShUXFBFXVNQgGDaBsCeEJGSbJLOemfv3x3nm5Mw+mZnMmTPn/bquueZ57uc+z/mek3hd5OP3vp9IKSFJkiRJkiQNVlXqAiRJkiRJkjQzGRxJkiRJkiRpWAZHkiRJkiRJGpbBkSRJkiRJkoZlcCRJkiRJkqRhGRxJkiRJkiRpWAZHkiRJkiRJGpbBkSRJmrSIWBARHRGRIuJVo8x7IJuzLSLqR5hzdTYnRcSqbGxV0dhYP8O95uJR6rltsp+/6H5HF73n6VN1X0mSpFKpKXUBkiRpVngFUA/cD/wVcMUoczuBhcALgG8UX4iIJcDZ2ZyGoktbgREDKeBk4C3APcCWYa7/bUR8KqW0afSPMWkXAruBDvLfwy8P8PtJkiQdUJFSKnUNkiSpzEXEzcB24GrgP4EjUkr3DTPvAWAP0AdsSCmdPej6O4B/Bb4FvBQ4LKX0wBjvvQS4CZgPnJpSuj0bX0U+yFoLnARcmlJ6w3D1pJSO25/PO0IdtcAG4HvALuAiYFlKafdk7y1JklQqLlWTJEmTEhFPAtYAXwK+CuTId9uM5v8BZ0TEwYPGXwt8n+G7hoZ771rgm8By4LX9odEgvyUfRP1VRDx+PPedoOcDB5H/Hr4INAMvHzypaAnd+yPiZRFxS7bMb31EvDabc0hEfDMitkfE7oj4ckTMHXSfoyLivyLi9mxOe0TcFBGvG+H9Rvp5f9Hcmoh4V0TcERGd2ZLCb0XE8aN8hnMi4vfZ/E0R8ZGIsKtdkqRZwuBIkiRN1oXku4iuSik9Rr7j5oKIGO2/M75Mvuvogv6BiDgVOBq4bD/e+z+B04APp5S+Ocq89wAB/Nt+3Ht/XUi+w+mXKaV1wM2MHqCdA1wC/C/wDvJL3C6LiFeQX+K2G/gH4ErySwE/Puj1zwCeRv77fgfwf4Ee4HMR8Z6ief3L/Ab/XJ9d31w09yvAh8h3Tr0D+CzwTOA3EXHCMJ/hbPJ/Xj8A3gb8Efh74J2jfG5JklRGXKomSZImLCIagE3A1Sml12Rj5wLfBs5OKf1g0PwHyJaGRcRVwHEppcdn1y4l37Wzknwg9GZGWaqWdedcBlwHnJVS6h10fRX5IOfTKaW/zu7/euApKaUbB9czye/hYOAh4F9SSu/Pxt6afY5jUkp3DlNXe3btwWy8FXgYqAP+PqX0saLX/C/5oGlhSmlPNtacUto7qI4q4CfACcDilFLPCPWeQ35Z4dXAX6SUUkQ8B/gR+aDqvJT9R2JEPJH8UsDfpJROH+YzHNv/ZxQRAdwKLEopLduf71CSJM1MdhxJkqTJeDHQQn55Vr9ryHe5jLVc7TLgyIh4akQ0kl/WdUVKKTfWm0bEk4HPAA8A5w8OjUbwfvJBx7+PY+7+eg35/666vGjsK+Q7gEb6Hr7dHxoBpJS2AneR78T69KC5vwRqgVVF8wuhUUQ0RMQi8puO/wiYBxw13JtGxBrga+Q7ol6Z9v2/iC/Kfl9cNEZK6Y/Ad4HTsnBr8Gd4oGhuAn4KLI2IOSN8bkmSVEYMjiRJ0mRcSD4k2hARR0TEEcCh5MOLF0TE4lFe+0Py3UqvBV5CPuz4f2O9YUQcRH55Vx/w4pTStvEUmlJ6hHwH0OkR8fwR7j0nIpYO+qkbo54gHw6tA6qKvocW4FfAq0bY82fI5uHADmBTSqlrmHGARYNq/Y+IeIj8U9weI/9ncXE2ZcEwtS4nv7RtJ/D8lFJ70eXDyH+ndw5+HXB70ZyxPkP/n8eiYa5JkqQy48aFkiRpQiLiMPL73wRw9wjTXkk+rBkipdQbEZcDbwKOBW4sXtI1wnvWAN8AVgCvSindvJ9lfxh4A/CvEfH9Ya7/PfC+QWPPBH42yj2fDjwuO75nhDnnkF++V2ykLqnRuqei6Pir2X0vBX5BPrDpJb/v0NsY9H8QRkQz+c6h+cBpKaVNo7zPeI23VkmSVKYMjiRJ0kS9lnw48HryHSyD/Qv5Tpxhg6PMZcC7gFPJP75+LB8jvyH0JSmlL+9XtUBKqS0i/oX8RtMXDDPlcuCGQWN/HOO2fwV0Aa8m37Ez2H+T78waHBxNWES0kA+NrkgpvXHQtWcPM7+K/PK0JwLnZsvPBruPfNh0NPnuqWLHZL/vn2TpkiSpzBgcSZKk/ZYFEa8Bbk0pfX6EOccC74+IJ6eUfj/cnJTS3dkm0guB/xnjPV8FvIX8fj9vn0T5/wW8FfgA+X2D9hTVcx/DL78aqab55JfZ/SildOUIc55Hfrnasinq8oF9nT4DunoiYhnwumHmf4z8xuNvSyl9b4R7fpt899d7IuIvizbHPg54AXBDtg+TJEmqIAZHkiRpIs4g//SzL4wy5yryG1JfCAwbHAGklC4Z680i4njyS7L6yD8J7Lz81kLD+nUWAI30ft0R8X+BK7Khce2RNILzgUbyn3UkV5EP2S4g/6j7SUsp7Y6IHwGvjIgO8t/voeSX4d3PwL2QziIflN0BPBYRrxx0u3UppXUppesi4krgPGBBRHwPWEr+6XadwN9MRe2SJKm8GBxJkqSJuDD7/b8jTUgp3RYRd5MPed6WUuqYxPudCDRkx/8xxtzXMnbX0FfIdy2tmURNkP8ecsB3RplzHbA7q2tKgqPMK7P7PZ98KHUP8F7yT3Ir3mR8Sfb7GPaFZcU+wL6laa8A/kA+6PoosBf4OfB/U0q3TmHtkiSpTETR01YlSZIkSZKkgqqxp0iSJEmSJKkSGRxJkiRJkiRpWAZHkiRJkiRJGpbBkSRJkiRJkoZlcCRJkiRJkqRh1ZS6gLEsXrw4rVq1qtRlSJIkSZIkzRo33XTTYyml1rHmzfjgaNWqVaxdu7bUZUiSJEmSJM0aEfHgeOa5VE2SJEmSJEnDMjiSJEmSJEnSsAyOJEmSJEmSNCyDI0mSJEmSJA3L4EiSJEmSJEnDMjiSJEmSJEnSsAyOJEmSJEmSNCyDI0mSJEmSJA3L4EiSJEmSJEnDMjiSJEmSJEnSsAyOJEmSJEmSNCyDI0mSJEmSJA3L4EiSJEmSJEnDMjiSJEmSJEnSsAyOJEmSJEmSNCyDI0mSJEmSJA1rzOAoIlZGxE8j4o6IuD0i3pqNL4yI6yLinuz3gmw8IuKSiFgfEesi4klF97ogm39PRFxw4D6WJEmSJEnS1Nm+t5tcb1+py5h24+k4ygFvTykdA5wKvDkijgHeDVyfUloNXJ+dA5wFrM5+LgI+A/mgCXgfcApwMvC+/rBJkiRJkiRpJjv7E7/kPf97a6nLmHZjBkcppU0ppT9kx7uBO4HlwLnAl7JpXwJemB2fC1ye8m4EWiJiGfBc4LqU0vaU0g7gOuDMKf00kiRJkiRJU6yvL7Ftbxetc+tLXcq02689jiJiFXAC8FtgSUppU3bpUWBJdrwceLjoZRuysZHGJUmSJEmSZqzt7d309CYWzzE4GlFEzAGuAv42pdRWfC2llIA0VUVFxEURsTYi1m7dunWqbitJkiRJkrTf7n50NwBHLplb4kqm37iCo4ioJR8afSWl9L/Z8OZsCRrZ7y3Z+EZgZdHLV2RjI40PkVK6NKV0UkrppNbW1vF+FkmSJEmSpCnX0dMLwLzGmhJXMv3G81S1AL4A3JlS+ljRpe8A/U9GuwC4umj81dnT1U4FdmVL2q4FzoiIBdmm2GdkY5IkSZIkSTNWri+/yKoqosSVTL/xRGVPBV4F3BoRt2Rj/wB8CLgyIi4EHgRell27BjgbWA+0A68FSCltj4h/Bn6fzftgSmn7lHwKSZIkSZKkA6Q3C45qqg2Ohkgp3QCM9M08a5j5CXjzCPe6DLhsfwqUJEmSJEkqpUJwVFV5wdF+PVVNkiRJkiSp0vQHR9VVlRejVN4nliRJkiRJ2g/9exxVV+AeRwZHkiRJkiRJo+jt6wOgugL3ODI4kiRJkiRJGkXOPY4kSZIkSZI0nL7CHkcGR5IkSZIkSSriHkeSJEmSJEkaVuGpau5xJEmSJEmSpGLucSRJkiRJkqRh7enMURVQW115MUrlfWJJkiRJkqT9cP+2vRyysKkig6OaUhcgSZIkSZI0Ez28vZ3//PE97GrvYV5jbanLKQmDI0mSJEmSpGG866p1/PrebQCcctjCEldTGpXXYyVJkiRJkjQO/ZtiAzTVVZewktIxOJIkSZIkSRpGb1Fw9GhbVwkrKR2DI0mSJEmSpGHc9OCOwvHxy+eVsJLSMTiSJEmSJEkaJKU04PxVp64qTSElZnAkSZIkSZI0SEdP74DzRvc4kiRJkiRJEsCO9p4B5831BkeSJEmSJEkC9nblBpy3NNaVqJLSMjiSJEmSJEkaZHBwVKlL1WpKXYAkSZIkSdJM09Gd3+PoyxeewgmHtJS4mtIxOJIkSZIkScqklPjXa+6ksTbfYdTSVEtzfeXGJ5X7ySVJkiRJkgbp7Onjc7+8v3BeqUvU+rnHkSRJkiRJUqa7t2/AeXNdZffcGBxJkiRJkiRlunK9A86b6u04kiRJkiRJqni7O3t43iU3DBhrqjU4kiRJkiRJqngPbmtn6+6uwnldTRU11ZUdnVT2p5ckSZIkScq0dfYMOO/O9Y0ws3IYHEmSJEmSJAG7O3OlLmHGMTiSJEmSJEkC2jp6xp5UYQyOJEmSJEmSgLZBHUcHza0vUSUzh8GRJEmSJEkS+aeqFXvJiStKVMnMYXAkSZIkSZIEtHUM7DiqrfAnqoHBkSRJkiRJEjC046iuxtjEb0CSJEmSJAloGxwc2XFkcCRJkiRJkgTQ3t3LCYe08NJsb6OGWmMTvwFJkiRJkiSgO9dHXXUVPb19ADTV1ZS4otIzOJIkSZIkSQJ6evuoq6mipzcB7nEEBkeSJEmSJEkA9PQmaqurqM+WqBkcgT1XkiRJkiRJ5DuOaquD9559NHPra3jqEYtLXVLJGRxJkiRJkiQB3b191FZXsWhOPR8497hSlzMjjNlzFRGXRcSWiLitaOx/IuKW7OeBiLglG18VER1F1z5b9JoTI+LWiFgfEZdERByYjyRJkiRJkrT/erLgSPuMp+Poi8CngMv7B1JKL+8/joiPAruK5t+bUlozzH0+A7we+C1wDXAm8IP9L1mSJEmSJGnq9eQStdX2uRQbM0ZLKf0C2D7ctaxr6GXA10a7R0QsA+allG5MKSXyIdQL979cSZIkSZKkqbOlrZO1D+Rjj46eXuprqktc0cwy2f6r04HNKaV7isYOi4ibI+LnEXF6NrYc2FA0Z0M2NqyIuCgi1kbE2q1bt06yREmSJEmSpOE975M38JLP/oZcbx+7OnpY0FxX6pJmlMkGR+czsNtoE3BISukE4O+Ar0bEvP29aUrp0pTSSSmlk1pbWydZoiRJkiRJ0vC27u4CYHP2e5HB0QATfqpaRNQALwZO7B9LKXUBXdnxTRFxL3AksBFYUfTyFdmYJEmSJElSyf3hwR0AHLlkbokrmVkm03H0bOBPKaXCErSIaI2I6uz4cGA1cF9KaRPQFhGnZvsivRq4ehLvLUmSJEmSNGU2t3UC0NJUW+JKZpYxg6OI+BrwG+DxEbEhIi7MLp3H0E2xnwasi4hbgG8Cb0wp9W+s/Sbg88B64F58opokSZIkSSqh/PO78rbuyS9Va6h1c+xiYy5VSymdP8L4a4YZuwq4aoT5a4Hj9rM+SZIkSZKkA+Jrv3u4cPzY7m4A6msmux307OK3IUmSJEmSKtLaB7YXjn97/zbAjqPBDI4kSZIkSVJFOubgfQ+C37CjA4CGWqOSYn4bkiRJkiSpInXl+oaMNdTYcVTM4EiSJEmSJFWkHXu7h4xVVUUJKpm5DI4kSZIkSVLFuXLtw3z+hvupczPsUfntSJIkSZKkivPRH90FQHGD0YmHLihRNTOXwZEkSZIkSao4ud4EQG9fKox9/aJTS1XOjGVwJEmSJEmSKs6KhU0AvPO5RxXGatzfaAiDI0mSJEmSVHFqqoI/e9wiXnHqIYWxCIOjwQyOJEmSJElSxens6aWhtppqu4xGZXAkSZIkSZIqTj44qqK2ymhkNH47kiRJkiSp4nT29NFQU02VHUejqil1AZIkSZIkSdOtK9dLfW01AF953Sk8tL29xBXNTAZHkiRJkiSp4nT29NFQm1+I9dQjFvPUEtczU7lUTZIkSZIkVZz+zbE1OoMjSZIkSZJUUXp6+8j1JRoNjsZkcCRJkiRJkirKzvYeAFqaaktcycxncCRJkiRJkirK9r3dACxsritxJTOfwZEkSZIkSaoo923dA8DKBU0lrmTmMziSJEmSJEkVZePODgBWLW4ucSUzn8GRJEmSJEmqKNv2dlNbHcxrqCl1KTOe35AkSZIkSaoIH7/ubo5eNo97Nu/m4JZGIqLUJc14BkeSJEmSJGnWa+vs4RPX30NVwEFzGzj18IWlLqksuFRNkiRJkiTNCtfe/ii/vvexYa99Y+0GAPoSPNrWyVHL5k1naWXLjiNJkiRJkjQrvOGKmwB44EPPG3LtX75/x4Dz1QfNmZaayp0dR5IkSZIkaVZZv2XPkLFnHNk64Lypzl6a8TA4kiRJkiRJs8qzP/bzAecpJf706O4BYw21RiLj4bckSZIkSZJmnZ7evsLxz+/eyqZdnQOuL2yum+6SypLBkSRJkiRJKnsppQHnxcHRzQ/tHDL/0EXNB7ym2cAFfZIkSZIkqex15foGnPfkEmRNRZ+4/p7C+If/4ngOmtcwnaWVNYMjSZIkSZJU9jq6ewecd/f2DZlz5RuewsmHLZyukmYFl6pJkiRJkqSy194zMDhq6+wB4J7Nuzn24HkAhkYTYHAkSZIkSZLK3uCOoxd88gYAnvPxX3D7I20cPN/laRNhcCRJkiRJksrero7uAed7BwVJ1dUxneXMGgZHkiRJkiSp7D20vX3U67VVRiAT4bcmSZIkSZLK3oPb2omAtz37yGGvV1fZcTQRBkeSJEmSJKnsbd3dxYKmOp64cv6w1w2OJsbgSJIkSZIklb29XTnm1NdQWz181DHSuEbntyZJkiRJksrab+/bxnfXbaK5vmbEzqLuXN80VzU71JS6AEmSJEmSpMl4+aU3AjCnvpqRFqRt29s9whWNZsyOo4i4LCK2RMRtRWPvj4iNEXFL9nN20bX3RMT6iLgrIp5bNH5mNrY+It499R9FkiRJkiRVspqqKqqKOo5SSoXjrlxvKUoqe+NZqvZF4Mxhxj+eUlqT/VwDEBHHAOcBx2av+a+IqI6IauDTwFnAMcD52VxJkiRJkqQpsXFnB8Ur1bpcnjZpYy5VSyn9IiJWjfN+5wJfTyl1AfdHxHrg5Oza+pTSfQAR8fVs7h37XbEkSZIkSdIwIiBiX3LU2WOX0WRNZnPsv46IddlStgXZ2HLg4aI5G7KxkcaHFREXRcTaiFi7devWSZQoSZIkSZIqRXXEgD2OenrTiHM1PhMNjj4DPA5YA2wCPjplFQEppUtTSiellE5qbW2dyltLkiRJkqRZpHgfo2OXz+eQhU2F894+g6PJmtBT1VJKm/uPI+JzwPey043AyqKpK7IxRhmXJEmSJEmakP59jFYfNIcPvfh4mutrePqRrezs6CHX5x5HkzWhjqOIWFZ0+iKg/4lr3wHOi4j6iDgMWA38Dvg9sDoiDouIOvIbaH9n4mVLkiRJkiTBfVv3AnDR0w6nuT7fH1NTFfT29WFuNHljdhxFxNeAZwCLI2ID8D7gGRGxBkjAA8AbAFJKt0fEleQ3vc4Bb04p9Wb3+WvgWqAauCyldPuUfxpJkiRJklRRNuxoB+CopfMKY9VVQa43Deg4iiGv1HiM56lq5w8z/IVR5l8MXDzM+DXANftVnSRJkiRJ0ij2dOUAmNuwL+KoqQ56+9KAPY7c7WhiJvNUNUmSJEmSpJLa3ZkPjuYUBUfVVVX09iVybo49aQZHkiRJkiSpbH38x3cDgzqOqoLcoI4jl6pNjMGRJEmSJEkqWzvbewCor6kujFVXxZCOo9effvi01zYbjLnHkSRJkiRJ0kzzjbUPD9gQu1hNVbBxZwf/+O1bAbjsNSfx50ctmc7yZg2DI0mSJEmSVHbe8c11I1675eGdANy2sQ2AhqJuJO0fl6pJkiRJkqSyktLATa9fd9phA87/9OjuAedVVe5wNFEGR5IkSZIkqax09PQOOD90UdOo8+fUu+BqogyOJEmSJElSWdnTlRtwXlczerzR0lR7IMuZ1QyOJEmSJElSWdnbNbDjqLZ6YLzx1metHnC+bH7jAa9ptjI4kiRJkiRJZWXvGB1Ha1a2FI7/4kkrqHaPowkzOJIkSZIkSWVld+eg4GhQx1FxULRm5fxpqWm2MjiSJEmSJEllZXDHUW3NyMGRT1SbHIMjSZIkSZJUVvZ2DwyO6gd1HFXFvrCoOgyOJsPgSJIkSZIklZWxnqpW3HH0nGOWTEtNs5XBkSRJkiRJKitDlqoN2eMo/3t5SyOL5tRPV1mzksGRJEmSJEkqK3u6egecD+04yp831Bp7TJbfoCRJkiRJKit7Bj9VbVBw1L9QraG2epoqmr0MjiRJkiRJUlnZ25Vjybx9S9DqBi1V68r1AdBocDRpBkeSJEmSJKms7OnO0VxfUzgf3HHU0ZNfytZYZ3A0WQZHkiRJkiSprOztyjGnKDgavDn2/MZaAI5bPn9a65qNasaeIkmSJEmSNHPcuamNQxc1F84HdxytWdnC/1x0KiceumC6S5t1DI4kSZIkSVLZ6OjuZXNbF5vbugpjtdUxZN4phy+azrJmLZeqSZIkSZKkstHenRsyNnhzbE0dv1lJkiRJklQ2+p+Y9r7nH1MYixjacaSpYXAkSZIkSZLKRncWHPVvgK0Dy+BIkiRJkiSVje7efHA0eENsHRhuji1JkiRJkspGf8dRXXUVV1x4Mus27CpxRbObwZEkSZIkSSobXbleAOprqzl9dSunr24tcUWzm31dkiRJkiSpbHQVdRzpwPNbliRJkiRJZaOwVM09jqaF37IkSZIkSSob/cFRvcHRtPBbliRJkiRJZaPL4Gha+S1LkiRJkqSy8NieLt7ytZsBl6pNF79lSZIkSZJUFh7a3l44NjiaHn7LkiRJkiSpLHR29xaO62uqS1hJ5TA4kiRJkiRJZaGjJx8cHbV0LguaaktcTWUwOJIkSZIkSWWhsye/MfYnzjuBiChxNZXB4EiSJEmSJJWF/o6jxlqXqU0XgyNJkiRJklQW+oOjhjrjjOniNy1JkiRJkspCV39wZMfRtBkzOIqIyyJiS0TcVjT2kYj4U0Ssi4hvRURLNr4qIjoi4pbs57NFrzkxIm6NiPURcUm4GFGSJEmSJO2Hjm6Xqk238XQcfRE4c9DYdcBxKaUnAHcD7ym6dm9KaU3288ai8c8ArwdWZz+D7ylJkiRJkjSijp5eaqqC2moXUE2XMb/plNIvgO2Dxn6UUsplpzcCK0a7R0QsA+allG5MKSXgcuCFEytZkiRJkiRVoq27u2iss9toOk1FRPdXwA+Kzg+LiJsj4ucRcXo2thzYUDRnQzYmSZIkSZI0Ljc/vJMTD11Q6jIqyqSCo4h4L5ADvpINbQIOSSmdAPwd8NWImDeB+14UEWsjYu3WrVsnU6IkSZIkSZolNuxoZ/VBc0pdRkWZcHAUEa8BzgFekS0/I6XUlVLalh3fBNwLHAlsZOBythXZ2LBSSpemlE5KKZ3U2to60RIlSZIkSdIskevto7Onj7kNtaUupaJMKDiKiDOBdwIvSCm1F423RkR1dnw4+U2w70spbQLaIuLU7GlqrwaunnT1kiRJkiSpIuztyj9Rrbm+psSVVJYxv+2I+BrwDGBxRGwA3kf+KWr1wHX5HIgbsyeoPQ34YET0AH3AG1NK/Rtrv4n8E9oaye+JVLwvkiRJkiRJ0oj2dOef0TWn3s2xp9OYwVFK6fxhhr8wwtyrgKtGuLYWOG6/qpMkSZIkSQL2duWDIzuOptdUPFVNkiRJkiTpgNrd2d9xZHA0nQyOJEmSJEnSjNffcWRwNL0MjiRJkiRJ0oy3s6MHwKeqTTODI0mSJEmSNOPdv3UvEXDooqZSl1JRDI4kSZIkSdKMkuvt439+/xBdud7C2I72bubU19BQ61PVppMLAyVJkiRJ0ozy4zu38K6rbuXBbe288RmP49frH2NPV4657m807fzGJUmSJEnSjPLYni4Atu/t5iM/vIsrbnwQgJYm9zeabi5VkyRJkiRJM0p7d/4Jak11NTy8o70wvrO9p1QlVSyDI0mSJEmSNKN05/oA+PKND9Lbl0pcTWUzOJIkSZIkSTNKVxYcdff28eC29jFm60AyOJIkSZIkSTNKf3AE8ND2fcHRk1ctKEU5Fc3NsSVJkiRJ0ozS1dM74PyFaw7mVU85lOOXt5SoosplcCRJkiRJkmaU4o4jgA+/5AnU11SXqJrK5lI1SZIkSZI0o+zpyhWOj1o619CohAyOJEmSJEnSjNLWuS84Wjq/oYSVyOBIkiRJkiTNKLs6egrHS+YaHJWSexxJkiRJkqQZIaXE0z/yswFPUltix1FJ2XEkSZIkSZJmhI9ce9eA0Ajg6Ue2lqgagR1HkiRJkiRphvjirx8oHB+ysInv/c1pzGuoLV1BsuNIkiRJkiTNDIcuai4cf+7VJxkazQB2HEmSJEmSpBlh4452XnXqofzzC48rdSnK2HEkSZIkSZJK7pLr76GtM8fC5rpSl6IiBkeSJEmSJKnkPnbd3QC0NLk8bSYxOJIkSZIkSSXV25cKx7XVRhUziX8akiRJkiSppLpzfYXjrqJjlZ7BkSRJkiRJKqm+tK/j6FlHHVTCSjSYwZEkSZIkSSqp/tjovWcfzarFzSWtRQMZHEmSJEmSpJLq7ziKKHEhGsLgSJIkSZIklVTKtjWqMjmacQyOJEmSJElSSfV3HFUPVuYRAAAgAElEQVSZG804BkeSJEmSJKmkCsGRydGMY3AkSZIkSZJKqi/bHTtcqjbjGBxJkiRJkqQptbO9m1s37Br3/ORStRnL4EiSJEmSJE2p131pLc//1A309PaNa35/x5GbY888BkeSJEmSJGlK/eGhHQBs3NExrvlujj1zGRxJkiRJkqQp1d9BtKcrN875+RcEJkczjcGRJEmSJEk6ILpyveOalwqbYx/AYjQhBkeSJEmSJGnKbN3dVTju7BnfHkfJPY5mLIMjSZIkSZI0ZT74vTsKx+PtOCrscWRKMeP4RyJJkiRJkqZMe9G+RuPtONq3ObYdRzPNuIKjiLgsIrZExG1FYwsj4rqIuCf7vSAbj4i4JCLWR8S6iHhS0WsuyObfExEXTP3HkSRJkiRJ0yGlxMadQ5+atqujh+UtjcD+dBzlf4fB0Ywz3o6jLwJnDhp7N3B9Smk1cH12DnAWsDr7uQj4DOSDJuB9wCnAycD7+sMmSZIkSZJUXq648UGe+qGf8NLP/row1tPbxy0P76R1bj0wvo6jn/5pC/90db5PpcrcaMapGc+klNIvImLVoOFzgWdkx18Cfga8Kxu/PKWUgBsjoiUilmVzr0spbQeIiOvIh1Ffm9QnkCRJkiRJ0+7qWx4B4PcP7OBNX7mJ1QfNpTPXS64vUZMlQF09Y3ccvfaLvy8cu1Rt5pnMHkdLUkqbsuNHgSXZ8XLg4aJ5G7KxkcYlSZIkSVIZSSmxqWiZ2jW3Psonrr+HH972KADHLZ8PwEPbBy5l6+zp5bQP/4Sf3bUFgC1tnQOu23E080zJ5thZd1GainsBRMRFEbE2ItZu3bp1qm4rSZIkSZKmwNoHd/DIrs4h4w9uawfg3WcdBcBlv7qfzUXh0MadHWzY0cF7v5VfmnbLwzsHvN49jmaeyQRHm7MlaGS/t2TjG4GVRfNWZGMjjQ+RUro0pXRSSumk1tbWSZQoSZIkSZKm2t2bdwMU9jIarKG2unD84v/atwdSZ7Z0bUd7NwAXXXHTgNcZG808kwmOvgP0PxntAuDqovFXZ09XOxXYlS1puxY4IyIWZJtin5GNSZIkSZKkMvLIzg5qqoIzj1065Nqrn3IoAP3NQxt3dvDb+7bRlevlkZ357qP27uH3PnKPo5lnXMFRRHwN+A3w+IjYEBEXAh8CnhMR9wDPzs4BrgHuA9YDnwPeBJBtiv3PwO+znw/2b5QtSZIkSZLKxyM7O1k6v4EHtu0dcm1uQ/45XPf/2/M47YjFALz80ht55zfX8frL1xbm9fTmn7j21metLoxVTcmGOppK432q2vkjXHrWMHMT8OYR7nMZcNm4q5MkSZIkSTPOQ9vbObilkd2duSHX5jXUFo67c32F4/6nsPW76cEdACyaU1cYCxerzThmeZIkSZIkadzWb9nDTQ/u4PDFzVQP8xi09Vv2FI47c8MvSQN439W3A7CweV9wtHrJnCmsVFNhXB1HkiRJkiRJKSWe/bGfA3DioQuoqopC51C/g+bt2zC7fzPs4Ty0Pf8EtoVNdSxoqmVBcx0rFjQdgKo1GQZHkiRJkiRpXP64YVfh+KB5DfzTOceweE49ZxyzhHM+eQMAb/nzfXsWdQwKjlYtauK7bzmN5378FzyyK79R9rKWRtb+43OmoXpNhEvVJEmSJEnSuHzxV/cXjk84pIWG2mr+7jlHctzy+YXxhtrqwvG7zjxqwOufe+xS5jbUMr9p3/K05S2NVFfFsMveVHp2HEmSJEmSpDF98Lt38O2iDa7n1g+MFH70tqexrqgjCeCcJxzMmccu5Yj3/gCABdl+Rl1ZJ9J/vPSJ1NXY0zKT+acjSZIkSZLGdFlRtxFAxMAOoSOXzOUlJ64Y8rqa6io+/ZdPyubkN79u784HR8vmNxyIUjWF7DiSJEmSJEnj9uITlvOmZx6xX685+/il/PKdz2Tlwvzm10vm1fNoWydLDY5mPIMjSZIkSZI0qpRS4fjglkaOOGjOfr0+IgqhEcBnX3Ui31+3icMXN09ZjTowDI4kSZIkSdKounJ9AJx46ALe9MzHTfp+y+Y38rrTD5/0fXTguceRJEmSJEka1Z6uHADPf8IymursQakkBkeSJEmSJGlUm3Z2ArCspbHElWi6GRxJkiRJkqRh7Wrvoa8vsWFHOwDLDY4qjv1lkiRJkiRpiHu37uFZH/05i+fUcfJhCwEGbHCtymDHkSRJkiRJGmLdhp0APLanm2tufZRl8xuY31hb4qo03QyOJEmSJEnSEDvbewacL2iqK1ElKiWDI0mSJEmSNMTW3V0Dzr/8ulNKVIlKyeBIkiRJkiQNsXV3F4211YXzhc12HFUigyNJkiRJkjTElt1drF4yB4AXrjm4xNWoVHyqmiRJkiRJGmJzWyfLWxq5/QPPpb7GvpNK5Z+8JEmSJEkaoKO7l7s272b1krk019dQU218UKn8k5ckSZIkSQO0dfaQEqxc2FjqUlRiBkeSJEmSJGmA3Z05AObUu8NNpfNvgCRJkiRJFe4fvnUrT161gGcfvYSLv38nX//9w4DBkQyOJEmSJEmqaN25Pr7624f46m8fGnKtpam2BBVpJnGpmiRJkiRJZWr9lt2s37JnUvfYuqdrxGtL5jVM6t4qf3YcSZIkSZJUpp79sV8AcP3bn87CpjoWNNeN63UpJX561xbuf6ydPdl+Rv3+/S+ewDuvWgfA8hY3x650BkeSJEmSJJWhWzfsKhy//ktrue+xvVxx4cmcvrp1zNdee/ujvPHLfyicH7qoiZ+/45mF86OXzeO45fOIiKktWmXHpWqSJEmSJJWhH9+5uXB832N7Abjujs0jTR/goe3tA85b59QPOD9+xXxDIwEGR5IkSZIklaWbH945ZKynt29cr31sT/eA8xesOXhKatLsY3AkSZIkSVIZauvo4SmHL+I9Zx1VGNvb1Tvm61JKPLqrc8DY6oPmTnl9mh0MjiRJkiRJKkNtHT0snFPHnIZ92xdv3Nkx5utefdnv+M4fH2HZ/H1PTDvm4HkHpEaVP4MjSZIkSZLKUFtnD/Maalm5oKkwNp6lar+85zEAVizY98S0+Y21U1+gZgWDI0mSJEmSysyWtk4e29NNX1/iaUe28tZnrQagOze+PY4A1qxsAeCopS5T08hqxp4iSZIkSZJmkk//dD0A3133CB9+yRN423OOZP3WPfxpU9uor+vsye+BdMphC3nnmUfx6qesYk690YBG5t8OSZIkSZLKTP/SsotfdFxhrK66ip7eNOrrPn7d3QC8/Mkrqa2uYuXCplHnSwZHkiRJkiSViXUbdpLrS7R15phbX8OLTlhRuLZ9bzcPbW/nto27OPbgeUTEkNfftXk3AGcdt2zaalZ5MziSJEmSJKlMvOBTvwLg2UcfxLKWhgHXfn73VgDO+eQN/NM5x/Cb+7bxsZc9kbkN+za+7u1LPOmQFhrrqqevaJU1N8eWJEmSJKnM/PjOLbTOrR/x+ge/dwfX3bGZj/7o7gHje7pyNLunkfaDwZEkSZIkSTPY5rZOtrR1ktLA/Yt+d//2AeeNtUO7iLpyffxq/WNcfctG9nTl2NLWRXOdwZHGz78tkiRJkiTNQO3dOW7b2MbL/vs3LGqu47DFzQOu/+fLTxhw/sO/PZ2nf+RnA8auu+NRvva7hwaMnXr4ogNSr2anCQdHEfF44H+Khg4H/gloAV4PbM3G/yGldE32mvcAFwK9wN+klK6d6PtLkiRJkjSb/f03/sg1tz4KwLa93Wzb2z3g+jOPah1wfsgwT0h7bE/3kLE59e5vpPGbcHCUUroLWAMQEdXARuBbwGuBj6eU/qN4fkQcA5wHHAscDPw4Io5MKfVOtAZJkiRJkmarX9+7bcjY3PoadnflAGgatORsuKeoDafJPY60H6Zqj6NnAfemlB4cZc65wNdTSl0ppfuB9cDJU/T+kiRJkiSVvd2dPXzy+nvozvUVxo5fPr9w/IFzj+W0IxZzyfknDPfycZljcKT9MFV/W84DvlZ0/tcR8WpgLfD2lNIOYDlwY9GcDdmYJEmSJEkC/vvn9/Gpn67noHn1zGuoZWd7D/953hoOmlvPlt1dPK51Di9+0ooRX//tNz+VF376VwAsm9/Apl2dnHjoAr5+0am84xt/5Nu3PMLRy+ZO18fRLDDpjqOIqANeAHwjG/oM8Djyy9g2AR+dwD0vioi1EbF269atY79AkiRJkqRZoLs332n0rqtu5aHt7bzilEN4XOsc5jbU8rjWOWO+fs3KFmqr80vW+juV9nTmqK2u4u+f+3je8udH8LTVraPdQhpgKpaqnQX8IaW0GSCltDml1JtS6gM+x77laBuBlUWvW5GNDZFSujSldFJK6aTWVv9CS5IkSZIqw6/vfWzAeXXV+PYtKtbTmwB4/hMPBqClqRaAFQuaePsZj6emeqp2rVElmIqlaudTtEwtIpallDZlpy8CbsuOvwN8NSI+Rn5z7NXA76bg/SVJkiRJmhVu29g24Lyxdv+fgNZcV83e7l6OOGgOnzhvDSceumCqylMFmlRwFBHNwHOANxQN/3tErAES8ED/tZTS7RFxJXAHkAPe7BPVJEmSJEmVqjvXxz9++1YOXdTMG552eOGpaH/2uEWcf/IhdOX6OPv4pft936rsPo211Zy7xq2FNTmTCo5SSnuBRYPGXjXK/IuBiyfznpIkSZIkzQZfuOF+rly7AYAnrmjhni27AXjpSSsKy8wmIsuNaKzb/24laTCfwSdJkiRJ0jS69vZHWbGgkQ//8E+FsVd+4beF42OWzZ/U/Rtqq2nrzFEzgf2RpMEMjiRJkiRJmiZ9fYk3XHHTqHOWzKuf1Ht8+XWn8K2bN7KwuW5S95Fgap6qJkmSJEmSxmHb3u4hY08/cuDTxFuaJhf4HLlkLu8686jCnknSZBgcSZIkSZI0TXZ39gw4/+rrT+H9Lzi2cF5X4z/TNbP4N1KSJEmSpGmypys34PyIg+Zw2OJm/vtVJwLQYHCkGca/kZIkSZIkjcNv7t3G6y9fS29fmvA9dncODI5a5+T3Mzp66TyASd1bOhDcHFuSJEmSpHF4+5W38MiuTu5/bC9HHDRnQvfoX6r27rOO4nWnHVbYh2hZSwMAL3vyyqkpVpoiBkeSJEmSJI3D3IZa2NXJlrbOCQVH6zbs5I1f/gMAzzt+GTXV+xYB1VZXcfsHnktDbfWU1StNBYMjSZIkSZIG6e1LnPHxn3Pv1r089YhFvOeso+np6wNgR3vPGK8e3sXfv7NwPK+hdsj15nr/ia6Zx7+VkiRJkiQV+c292/jq7x7i3q17AfjV+m2c88kbCtd3dUwsOJrbsO+f4PMa/ee4yoN/UyVJkiRJKnL+524c9Xp3rndC910yr6Fw3L+3kTTT+VQ1SZIkSZL2Q0/vxJ58dnBL4xRXIh14BkeSJEmSJGW27ekacP7ZVz5pyJzu3r4J3btngq+TSsngSJIkSZKkzC0P7xxwfuZxywrHP3n70wHozk0uOPrh354+weqk6WdwJEmSJElSZrSNr5fMa6C2OibcOdTTm2iqq+aopfMmWp407QyOJEmSJEnKtI0SHDXVVVNbXTXh4Kg710dttf8MV3nxb6wkSZIkSZm2zhwANVXBv734eABOO2IxkH8SWm11VWGp2tbdXWzc2THmPVNK3PLwTrp7DY5UfmpKXYAkSZIkSTPFro4emuuquf2DZxbGPn/BSeztygdKy+Y3cHO2D9KTL/4xAA986Hmj3vMbN23gnd9cB8DB8xsORNnSAWPUKUmSJEkS0JXr5Qs33M/e7t4B4w211SyaUw/ku49u27iLlNK473vHI22F4xo7jlRm/BsrSZIkSRLw63u3jTlnQXMdfQm692Ofo/raff/0fmh7+4Rqk0rF4EiSJEmSJGDDjvx+Rb985zNHnNNYWw3AR35417jve++WvZMrTCohgyNJkiRJkoANO9qpq65ieUvjiHOqIv/78zfcP+77PtrWwZqVLZMtTyoJgyNJkiRJkoCbH9rJ6iVzqOpPh4aR6xv/3kb9Ht3VyVFL506mNKlkDI4kSZIkSQIe3LaXYw+eN+qcv3jSCgDmN9aOeb+e3j4+9ZN7eGxPN0t9mprKVE2pC5AkSZIkqdRSSmzd3UXr3PpR5y1oruO5xy7h2ts3j3nP1e/9QeF45YImPvKSJ1BXY/+GyovBkSRJkiSp4nX39tGX9m1+PZrr7hg7NBrs2OXzOGrp6N1M0kxk1ClJkiRJqnjduT6AcXUEDd7mKKWh+x71DZpkaKRyZXAkSZIkSap4heCoev//mdzR0wvAro4evvvHR+jrS/xy/WOF65f/1clTU6RUAi5VkyRJkiRVvPbufPhTVzP2UrW/fuYRfOqn6wvn2/d201RXw39cexdX3Pgg37p5Iz/50xYA/vF5R/O0I1sPTNHSNLDjSJIkSZJU8U7/958C41uq9oQV8wec72zvASCRX57WHxoBvPSklVNVolQSBkeSJEmSpIpWvEfReIKj47Pg6IkrW4B93Uq53qF7Hc2td6GPypvBkSRJkiSpou3Ngh8Y31PVls1v5IEPPY9/OucYANq7czy8vZ1bHt45ZG5VVUxdoVIJGH1KkiRJkirajr3dheOmurGDo37N9fm5Hd29haVu0mxjx5EkSZIkqeJ09vSyZXcnAD+87VEAaqqCEw9dMO57NNXmezHaizqWAH70tqcBcOiipqkoVSopgyNJkiRJUsV501f+wMkXX8+Ovd1cfM2dAHz+gpNoGMdStX6NWXdSe3eOY5bNA+B7bzmNI5fM5d9efDxXvuEpU1+4NM1cqiZJkiRJqjj9Tz474Z+vK4wdt3z+SNOH1VQIjnqprQ6e8fjWwj3OP/mQKapUKi2DI0mSJEnSrHTzQzt40X/9GoD/84zHccYxSzjhkOGXojXWVrN4Tv1+3b9/I+2P//huOnv6eO6xSyZXsDQDuVRNkiRJkjTr9PWlQmgE8Jmf3cvrL18LwK0bdg2Z/4pT9r9DqP+JaZ09fQBce/vmiZQqzWgGR5IkSZKkWWdztvF1seos6Hn+p24YMH766sW8+6yjpqUuqdwYHEmSJEmSZp3uXL4LqKWptjDW1pHjjkfahsx94ooWaqon9s/jM49dWjj+6EufOKF7SDPZpIOjiHggIm6NiFsiYm02tjAirouIe7LfC7LxiIhLImJ9RKyLiCdN9v0lSZIkSRqspzcfHH3w3OO46v88hdf82So6eno5+5JfFuYctzz/JLQFzXUTfp+6mn3/rD7r+KWjzJTK01R1HD0zpbQmpXRSdv5u4PqU0mrg+uwc4CxgdfZzEfCZKXp/SZIkSZIKenoTALVVwYmHLmRuw8BnQ11x4cn05PJzFk1BcFRXXVXYLFuaTQ7UUrVzgS9lx18CXlg0fnnKuxFoiYhlB6gGSZIkSVKFymXBUf8StOb6gcHRsvmNdOV6gcl1HNVnwVFLUy0RMeH7SDPVVARHCfhRRNwUERdlY0tSSpuy40eB/mcSLgceLnrthmxMkiRJkqQp09OXX6pWU50PcwYHR8311eT68uHSgqJ9kPZXfU2+y2jp/IYJ30OayWrGnjKm01JKGyPiIOC6iPhT8cWUUoqItD83zAKoiwAOOWT/H4koSZIkSapsucJStXy/xJz6gcvIFjTVcXjrHDbs6BgSKu2PRXPy3UoLJ9G1JM1kk+44SiltzH5vAb4FnAxs7l+Clv3ekk3fCKwsevmKbGzwPS9NKZ2UUjqptbV1siVKkiRJkirM2ge3A1CbdRw11u4Lh9b+47NpqK3mEy9fw7++6HgOX9w84fdpnVMPQE2VDy3X7DSpv9kR0RwRc/uPgTOA24DvABdk0y4Ars6OvwO8Onu62qnArqIlbZIkSZIkTYl//+FdAHTl8kvW6mv37UW0OAt7FjTX8ZenHDKpvYn679sfUEmzzWSXqi0BvpX9j6wG+GpK6YcR8Xvgyoi4EHgQeFk2/xrgbGA90A68dpLvL0mSJEnSALvaewrH/U9T69/EeqrjnRULmgD+f3v3HSdVdf9//HVmZntjl12WpS69g3QQgYgK2KLYYoktUZOo32hiEluMxko0JpZYosaWn11jBRRErCC9d1gWWNr2Xqad3x8zO+yyQ1/YBd7Px4MHc889994z+7izO/OZcz4fhmSmNPKZRZqHwwocWWuzgAFh2guA08K0W+Cmw7mmiIiIiIiIyL7klVcD8NvTujGwQzKwO4m1o5Ernw3umMy0W0bTs3VCo55XpLlojOTYIiIiIiIiIs1GXpkbgBGdds8CCs04OgIrynplJDb+SUWaCWXvEhERERERkePKg1NWAZCaEBVqcwVzEDX2jCOR450CRyIiIiIiInJcWbm9FCCUBBugbYsYWsRG8NCkfk01LJFjkpaqiYiIiIiIyHGlW6t4sgsqSImLDLUlREew5C/jm3BUIscmzTgSERERERGR44LH5weg0u3j3AFtmng0IscHBY5ERERERESk2fP5LWMencXHS7aF3T9vUyHd7p7Goi1F5JXVkFYnv5GIHDoFjkRERERERKTZq/L42FJYyS1vL6nXvrOkmkq3lyte+hGAC56djdvnJy1egSORxqAcRyIiIiIiItLs+Xw29Pivn64kNT6KYZ1SuPj5OWH7a8aRSONQ4EhERERERESaPY/fH3r8yg/Z++2vwJFI41DgSERERERERJo9n9/ut8//jetKYnQEfmsZ0anlURiVyPFPgSMRERERERFp9morptVyOgw+v+WVa4Zyz8cr6JGewM3juhLlcjbRCEWOTwociYiIiIiISLNXO+No0sC2dE6N45ejO+EwhugIJ9/3HNfEoxM5fqmqmoiIiIiIyDFq5updFFW4m3oYR4U3GDj6SY80/u+0bsRGuoiO0OwikSNNM45ERERERESOEYu2FHHBs7P5+8UDiIlwctObiwB49opBnNUvo4lHd2R5g1XVXA7NfxA5mhQ4EhEREREROUZc8OxsAP7w3tJ67Te+sYjsyWc3xZCOGm+wqprLaZp4JCInFoVqRUREREREmrG5WQXc+MZCSqs9TT2UJlVcGXj+LocCRyJHk2YciYiIiIiINGNPf7WB7zfkM3X5TgC6tYqntNrDrtKaJh5Z46uo8fLanGxaxkXys6EdQu0+v+WKl+YCKK+RyFGmwJGIiIiIiEgztrO0ut7281cOpktaPIUVbmas2sntHyxvopE1Hmstk6et4d/fZoXaagNHZdUe+t03PdQ+NDPlqI9P5ESmwJGIiIiIiEgzZa2loHz3zKJzB7ShS1o8ACnBWTn3f7qKCrevwbHPfr2BbUVVPDSp31Eb76H6el1evaARwD9mrGNHcRWJMRH12iNdyrgicjQpcCQiIiIiItJMbSuuoqjSwwPn9eHnIzqGStLXdd7AtkxfuSu0vau0mvs/W8WUZTsAeOC8vjiacV6g7cVVXPvK/ND2z4a0550FW3lq5voGfc/s2/poDk1EUOBIRERERESkWdqYV85pj38DQN+2SRhjiAhTUSzCYUIVxwCmr9oVChoB7Citpm2LmCM/4EM0P7sw9PjpywYyuGMy7yzYWq9P+5QYPvjNySTtMftIRI48BY5ERERERESaCWstne6cCsC1ozJD7b0yEvd6TITTQXGlh3/OWMd1ozvx8JTVAFw1siOvz9nMxtzyQwoc+f2WhVuKGNwh+YjOWHpk6hoA5t51GumJ0VhriY10ck7/DJLjIqmo8XLvuX2IcGqJmkhTUOBIRERERESkmaj27J459MoP2QBcd0qnfVYSiwjm/Hly5nqerLO86//GdeP1OZtZtaOUMd3TDmocPr9lzKOz2FZcxU8HtOGpywYe1PEHY2dpNRFOQ3piNADGGFbdP/GIXU9EDo5CtiIiIiIiIs1Ejbdhkus/n9N7n8fERzWcDzDjd2NIjY8EYPK0NQc9jm/W5bKtuAoI5Ew6Unx+izHwm7Fdjtg1ROTwKHAkIiIiIiLSTLi9gRlHLeMiD/gYV5hlZN3SEzDm0JeX7SjZHSxy7nF+j8/PKz9soqTKc8jnr3suayE6cu8zqkSkaSlwJCIiIiIi0kzUBANH8dGBWUR/GN99v8cM79wSgIEdWgDw23FdQ/t+f0bg+Hs/XnFQ46gNCg3q0CIUzAJYub2EbndP46+fruLFb7MO6pzhuH2Bc0cqf5FIs6UcRyIiIiIiIs1E7VK1m07tisfn57KhHfZ7zEntW5A9+eyw+2qTYr82ZzP3ntuHKct34PNbzh/Ytl6/KrePjXnlZKbGsSG3nEc/X0uE05AYE0FhhRuAihovl77wY+iY7SVV+x1bfnkNQx78krvP6sX1Yzo32O/1WQAlvhZpxhQ4EhERERERaSY+WrwdgKSYCCb0aX3Y5zu9d3ro8T9mrONfszYANAgcTXjiW7YUVgIQGUy27fFZolwOajx+rLVc+Nxsyqq9oWP+t2gbnVrGcf2YzntN3n33h8sBeGjq6rCBI09wxpECRyLNl16dIiIiIiIizURtYCcxOqJRzpcUE8HL1wypd+49lVR5QkEjoN7StEiXE7fPT3GlhzU7ywC466yeoYTcj89YR897Pg973pe/38QXK3ftc3y114pwHno+JhE5shQ4EhERERERaUIF5TW8+G0WVe7dFdVq8xU1hjbB5Wp7k5VXHrY90uXAaWBTfgUfLMoB4KWrhnDDmC589Yex9fpm3jGFO/+3vF7b/Z+t2u/Yamcc1c5yEpHmR0vVREREREREmsiMVbu4/vUFQGA5F8Cfz+6116Vfh6LtHoGj1PioetvbiwMV1D688WS2FFYSG+kiyuVgSGYy05bv5KMl23lwSmBsndPiAGiVEM1LVw3hjbmbmbU2D4C35m1hfJ90hmamEB/lIjHaRWm1lzvO7MnkaWvIzq8gMzWu3rW9/kCOI5dDgSOR5kqvThERERERkSPAWktJlYffvbOETfkVDfZXe3yhoFFdCVaWbVgAACAASURBVNGN+/1+QnQE7VN2B488Pj+zN+YD8Mf3lnLTm4tC/c47qS1n9E5nTPc0YiNdXDi4Xb1zdU6LDz0+vXc6r1w7LLQUDuDaV+bT994vKK50U1rt5coRHemdkQjA+Ce+pazaw3Wvzee79Xn4/ZZf/XchoKVqIs2ZAkciIiIiIiKNzO+39P/rdAb8dTofLt7GqX//Gl9wdk2t9bvCLxHrkBIXtv1wfHTjKM7s25qz+2VQUuXh8hfnMmPVLt5bmBPqk7ifgNXDk/qFbR/XM50nLz2pXtsNrwcCQv3aJTG6WyoQyGfU777pfLk6lyv/M4+s/PJQQC0xpnFyOolI41PgSEREREREpJHtLK2uV4EM4NXZ2aHH5TVe/j59LQCf3nwKF9WZ2TOic0qjj6dlfBTP/XwwaQm7l6nVznbqkBLL5cM7NFjCVuulq4bw8KR+XD68w17PP6ZbWr3tedmFAJzTPwNjDMM6NXxOXweXuNWOQUSaJwWOREREREREGllOUVWDtgc+W8XCzUUAvPBtFt+sCwRO2ibH0CW4BOy5KwZhzJFbtlVe423Q9tYNI3h4Uj8cjvDXPb13+j6DRgDJcZFMu2U0LeMi67XHRgZmMb37q5ENjqnNmzT5gn77TeAtIk1HgSMREREREZFG9mNWAQBTfzuaL38/lueuGATA/Z+u5P2FObw+JxuA8b3TSY6N4PrRnXj+54OY2Lf1ER3Xb8d1Y1TXlqHt5NiIBsmzD1WvjES6ttqdA2nyBfWXttUuWZt5W/2KbON6tWqU64vIkaGqaiIiIiIickLbVlxFy7jIRqtkZq3lrXlb6N8uiV4ZCRhj6NoqnsuGdeCteVtY+t5SAB69sD+XDG0PgMtpmNg3o1Guvy8dWsbyxnUjyLxjCg4Di/8yvlHPX1slbdoto+kVTIpd67+/HI61FmMM7/16JBc/PweA1LjwS+REpHlQ4EhERERERE5Yfr9l1OSvGNezFS9fM/SQz7OtuIq/TVtDQUUNA9sns6Okmv8b163esrMz+7bmrXlbQts9Wicc1tgPx/TfjSE+qvE/Dk6+oB/frMtrEDSqVfvzGJqZwgPn9aFtcsxel8iJSPOgwJGIiIiIiJywiqs8AHy1JjfU5vdbnpi5nnU7y/jDhB6h5Vf+4Gya6at2Mr5363oBjwc/W8W0FTsB+GFDAW1bxHDZsPb1rjWmexprHpjIef/6gbW7yuiZ0XSBo+7pR+ba3dIT6HaA575yZOYRGYOINC4FjkRERERE5ITj9fl56ftNTJ62JtSWlVfOa7OzGdG5JU/NXA9AYYWbd389Ep/f0uWuqaG+/7hkAKf3Tufpmeu5aHB7FgSTXte6fHiHsEmuoyOcfPG7MUfoWYmINL5DDhwZY9oDrwPpgAVesNY+aYy5D7geqK2teJe1dmrwmDuBXwI+4LfW2i8OY+wiIiIiIiKH5JUfsusFjQDGPf4NAK/N2RxqK6p0M3naGrLyyuv19fos/e+bDsCL320KtV8wsC35FW5+MarTkRq6iMhRdTgzjrzAbdbaRcaYBGChMWZGcN8/rbV/r9vZGNMbuBToA7QBvjTGdLfW+g5jDCIiIiIicoLamFfOrW8vYfm2Ev5348m8NjubX4zqxID2LfZ6zH++38QDn63i5C6BymK/GtuZf3+TFbbvRYPb8cmS7Tz/zcYG++ZnFzZoe/O64ZzcNfUQn42ISPN0yIEja+0OYEfwcZkxZjXQdh+HnAe8ba2tATYZYzYAw4A5hzoGERERERE5cZ0WnCEEcMGzswH4em0eS+9tWCmsyu3jx00FPPDZKgBmbyygT5tE7jyzFx1SYrn7wxWhvjERTu46uxcOA+8vzAl77feC7a9cM5Qfswq49fTuxEQ2TlU2EZHmpFFyHBljMoGBwFxgFHCzMeYqYAGBWUlFBIJKP9Y5LId9B5pERERERETCqvaEX7hQ5fbh9flxOR2htpJKDwPun96g7+XDOwBwxfCOXDG8I7PW5NIuOSaUDLvS7QsFlH46oA0/6ZHGmX0z6PWXzwG4ckRHTu3ZilN7tmrU5yYi0pw49t9l34wx8cAHwK3W2lLgOaALcBKBGUmPH8I5bzDGLDDGLMjLy9v/ASIiIiIi0ijeXbCVzDum4Pb6m3ooe/X5ih30vCcQvPnD+O68+6uRoX1un5+ud0+jtNoTant3wdaw59mzstipPVvRLT0BYwzGGOKiXCy65wyW3juepy4byAWD2tWbVXTnWT0b82mJiDRLhxU4MsZEEAgavWGt/R+AtXaXtdZnrfUDLxJYjgawDahbj7JdsK0Ba+0L1toh1tohaWlphzNEERERERHZhx0lVYx9bBZLtxYD8Kf3lwHw+pzsphtUGB5fIJD13x838+v/tyjUfvGQ9gzrlEL25LN587rhofb3F+RQXuMFdgeOpv9uDHPuHMd7vx7JqK4t6dsmab/XTYmLJCkmol7b2gcn8t2fTiU2UkWqReT4d8iBIxOoLfkfYLW19h912jPqdJsE1C4W/gS41BgTZYzpBHQD5h3q9UVERERE5PB9sWInmwsqeWveFmq8u5d/PThldb1ZO01p7c4yut09jcw7pnDPR4GPF+N6tiJ78tmkJ0aH+p3cNZXHLx4AwP2fraLvvV/wxtzNrM8t55qTM+menkBGUgxDM1N447oRh5yTKMrlpH1K7OE/MRGRY8DhzDgaBVwJjDPGLAn+Owt41Biz3BizDDgV+B2AtXYl8C6wCvgcuEkV1UREREREmpbPBv5/e/5W/vXVhnr7dhRXH5lr+i2jJn/FVS/PY2dJ+GsszynhrXlbsNby4eL6CxVuOa0bL18zNOxxFw5uV2/7nzPWAXDJkPbhuouIyH4cTlW17wETZtfUfRzzEPDQoV5TREREREQOzbfr8uiWHk9GUky99oLymtDjp4OBo1FdW/LDhgLyy2voQf08QI1hc0EF24qr2FZcxYhHZvLwpH6hRNVVbh9/+3wNr87OBuD79flMWb4DgPG90/n5iI4M65Syz/P3bZvIim2lAOSXuwHonh7f6M9DROREcNjJsUVEREREpHkqKK9h3a4ypq/cyVUvz+O8f/1Qb/836/J49uuN9dpS46O4fWIg6fO36/P4YGEOs9bkNuq4Squ99bbv+nA5VW4f363P49Z3FoeCRkAoaATwyAX9GNM9jeiIfS8xmzQwMOvo7rN6hdrqVlkTEZEDp2xuIiIiIiLHIWstwx6eic9vQ225ZTX4/Banw7BmZylXvxxIOXr58A6kxEbyr1kbGJqZTEwwMPPvb7JCx/5452m0ToqmMdR4AhkrXrl2KHd8sIxdpTWhEvd1dU6LIyuvIrS9Z5LqvfnFqEzG906nfUosxkC39MafNSUicqJQ2F1ERERE5Di0LKekXtCoVnZBIBDz0JTVoba+bZIYkpkMBErUR7kazujZmFfeaGOr8QYqpCVGu/jHJSc12P/YRf3Jnnw2r/8iUKD5bxf2Y+pvRx/wrCFjTCh59XWjOzO2uyo1i4gcKgWORERERESOUcWVbs59+ns+2iN5NMCcrILQ47Hd0/j05lMAOO3xb9hSUIkNxpR+NqQ9lwxpx9juabxy7VBuOrUr0RG7PyZcPbIjsDvgdCjW7Soj844p/LAhH9gdOIpyORnVNZUND50Z6ps9+WwuDiaybpccS/bks/nZ0A70bpN4yNcXEZFDp6VqIiIiIiLHoP/Oyeaej1cCcOs7S+iVkUjXVvE4HYaSKg9PzVzPoA4t+N+NowCodO/OK/STv8+iTYsYzh3Qhr9d1D/UfmqPVgBEBZeqOQzce24f3pq/lez8CnJLq2mVePDL1W7/YBkAV7w0l40Pn0WNN7BULcoVCFC5nA42PHQmYSZIiYhIE9OMIxERERGRY0yV2xcKGtWa8MS3dLlrKn6/5YbXF1Dp9vGbn3QN7Y+N3P2dsd9CTlEVmS1jw54/NjIQOPrDhB44HIZ2LWJ48btNDHt4Jpl3TMHj84f6VtR4uf39ZWTeMYVHpq7G7fXj3yMCVFnjCz2esnwHBcFKZ3WTXLucDiJd+ngiItLc6DeziIiIiEgTKa50Mz+7kMw7pvDNujwA3p2/lZMfmUl+ec1ej8srC+zrkBLLyM4t6+3rfNdU5m4qDDxOi6u37+x+GfX77rG/VoTTQfbks7kxGHjKyq+/TO3JL9eHHr+3YCvvLNgKwL+/zWLwgzO468PlAGwtrOTjJdtYu6uMgR1aAPDbtxZz7yeBoFfbFjF7fY4iItI8aKmaiIiIiMhR9vXaXB6Zuoa1u8pCbVe/PI9//mwAfwou67rh9QWhZWYQqJI26IEZFFV6SEuIAuD2iT05rVcrKt0+Vu8o5YqX5ob6T+zTmi5p8fWu+8wVg/jNthLOefp7AH46oO0BjfeK4R14Y+6W0PaCzYUUlNdQ7fWzYY+k2WXVXt6ev5XOaXE8PHVNqP2pSwfy+3eXMD+7CIA/BmcziYhI86bAkYiIiIjIUfD+whw+X7GTX43tzDWvzA/b53fvLA09XrSlmGqPL7Sc6425Wyiq9AC7Zxy1TooiOsJJdEQgyfTfLx7A63Oy2VlSzaMX929wfoBWwaATgPMAAzd/nNCDN+Zu4aZTu/DR4u38mFXI4Ae/3OcxdYNG43un0z4lln9fOYRXZ2cztnsagzsmH9C1RUSkaRlrm3cGuiFDhtgFCxY09TBERERERA7Jewu20qdNEmc99V299vYpMbRtEcNzVwzm399m8fw3GwFwOQy3nNaNx2eso2frBD79v1OIcDo49+nv2VJYyem90vlgUQ4PTerLFcM7HtKYut89jaGdknnjuhEHfExBeQ0tYiOZtmIHN7+5uN6+3hmJ3HVWL6at2MGVIzsy8YnAcx2WmcJDk/rSKTUOl1NZMkREmhNjzEJr7ZD99lPgSERERESkceWV1XDrO4vx+mwo31BdPVsn8OGNo4gJJqGu9vjoec/nAPxqTGd+d0b30DbAA+f35Z6PVjCxT2uev3LwYY+v2hNIVl03OfXB+HjJNm55ewmp8ZE8eH5fBnZIJv0Qqq2JiEjTUeBIREREROQostZijCGvrIahD4VfxvWfq4dwWq/0sPtWbS/lw8U5/GliTyKcDqYu38GNbyyq1+fDG09mYAct8RIRkcN3oIEj5TgSERERETlMFTVeBj84g2qPv8G+M3qnM2PVLlrERuw1aATQu00ivdv0Dm2f1S+DBX8+nQuenc2WwkpS4iIVNBIRkaNOgSMRERERkcO0NKe4QdDo3AFteOyi/uSV1TA/u5BbT+t20OdNjY/iy9+P5Zt1eZzaI62xhisiInLAlKFORERERE4o1lpe/WETmXdMweNrOEPoYFV7fFzzcv0qacMyU3j6soFERzhpnxLLoj+fwTWjOh3S+SNdDs7ona7k0iIi0iQ040hEREREjhk5RZVUuX10S084pOOttdz05iKmLt8JwHWvLeC1Xww7rDHd9MYi3D4/vxjVib+c2ztsH8cBlr0XERFpbvS1hYiIiIgcMy594UfO+Oe3VLq9PD1zPRc8+wO5pdUHdGyl28tjX6wNBY0AvlmXR43Xd8jjuf/TVcxck8uori2555xeh3weERGR5kpV1URERESk2ftgYQ7TV+3ki5W7wu5//9cjGZKZ0qDdWsvdH63gx40FZOVXhNr/+8thPP3VBuZtKiTCaXjy0oGc1S/joMbk81u63DUVgGX3jScxOuKgjhcREWlKqqomIiIiIseFl77L4sEpq/fZ56Ln5/DAeX2Y0Lc1rRKiQ+3Xv76AL1fn1us7oF0So7ulMaB9C/rfNx2Pz3LjG4v4340n89TM9Vw7qhNVbi/jeqYT6ao/Qf/jJdu45e0l9domX9BPQSMRETluKXAkIiIiIs3SKz9s4qmZ64mJcAIwaWBbrhrZkU6pcfzp/WVsyC1n6i2jGfzADCrcPu75eCX3fLyS728/lXbJsXywMCcUNPrpgDZsLqhg8oX96ZIWD9Ag2HPBs7MB+HptHhBIcP3G9cP554x1pCdG8/MRHfnDe0sbjLNLq/gj9jMQERFpalqqJiIiIiLNUuYdU0KP/zihBzed2jW0ba3F47OhGUFTlu3gpjcXhT3P+ofOJGIvFckKK9zM3pjPzW8uDrt/QLskluaU1GubNLAtfdok8ujna+mZkcAHvzl5r+cXERFprrRUTURERESOad1axbM+t5z4KBdn9m1db58xhkjX7kplZ/VrzWMX9eeP7y+r1+/Soe33GdRJiYvknP5tSE+Mxu8PBKIe+2It95/Xh9P/8W2DoFGXtDgeuaAf0RFOrhvduRGepYiISPOmwJEcl9xef4OcBEfDjpIqdpXWcFL7Fkf92iIiIsebKo+PCwa25fFLBmDMvsvZG2O4eEh7zuqXwa7SalrGRZEY49rvcbWG1kms/eb1IwAY0L4FS7cW07N1Ap/fOga314/LYXA4DuycIiIixwPNqZXjTkWNl773fcG9H6846tce/89vOf+ZH/D4/Ef92iIi0vh8fsuWgsqmHsYJq9rjIzrSecDBH4C4KBed0+JJio04qOPC+f0Z3YmJcPL0ZQMBiHQ5FDQSEZETjgJHctzZXFCJ2+vntTmbj/q1y6q9AHS7expLthYf9euLiEjjqPH6WL+rjMenr2XMY7PYXly1175+v2Xh5kL8/uadN/JYkltazSPTVpNf7g4lxm4KY7unsfqBiXRLT2iyMYiIiDQ1LVWTZm1TfgULsgu5eEj7/fZ9c+4W3pm/pV4uAq/Pj+soJKtcuLmIT5dur9c2a02ulqyJiBxjcooquf2DZfyYVYivTiAou6CCNi1iAKhy++h73xf4/JbLh3fgzblbAHj0ov5ccgB/r05UFTVernhpLn/9aR8GhPn7uDynhGtfnUd+ubteu0szfERERJqUZhxJs3bx83P44/vLqPH69tv3rg+XN0hgmVde02hjsdZy2Qs/8vj0tQ32/eG9pbw6OxsglFvpyZnrufrleY12fREROXwen58fNuSzZ1XZbcVV3PbuUsb9/Rt+2FBASlxkvf2XvziXG99YiNfn5/rXF4SCSrVBI4BnZm048k+gznhHTf6KF7/NorjSvf8DjhBrbYOf5bKcYi58bjYLNxfWa//5f+ayZGsx5z3zA//5fhMLNxdRWOEOzdS6/MUf6wWNUuIiGd87nXP6tznyT0RERET2SjOOpFnLDwZ+cktraJ8Su9d+WwvD55/475zN/Gliz8Meh99veX9RDnOyCpiTVcCG3HLO7p8RejNbUeMN9X3q0oH8+v8tBOCbdXmh9ie+XMcTX67n/V+PZEidBJwiInL0fLR4G398fxkXDGzLP352EhAIfox5dNbuYND1wzm5SyrTlu9gxqpdfL0uj8IKN1OX7+SUrjl8vyEfgGcuH8RNby7ivJPakBDt4v/9uIWrX57Hq9cOPaDcOtZapq/axffr87ltfHf8Fl7+fhPx0S4mT1vD0Mxk5mcX8eD5ffn5iI71jp2xcifbiqt4aOpqHpq6mg4psbx09RAqarz0aJ1AbOSRfYtXmyT68pd+ZOW2Uq4f05lfj+2C31quenkexZUeLnxuDgBRLgd/nNCDxVt2L+F+4LNVoccJUS76t0+irMbLmX1bc8XwjqzaUcL1ozsfdo4iEREROXxmz2+JmpshQ4bYBQsWNPUwpIlk3jEFYL/Bls9X7OTX/28hz/98cChoA4Gklr89rdthj2PepkIu+fecBu03jOnMC99mAdC/XRJPXzaQji3jOOfp71ixrRSA7Mln13suddtEROToKav28L9F27j3k5WhtiuGd+CN4Kwhp8Ow6J4zSIqJaHDs4i1FTHp2dmj7kQv6cenQ9izcXMTADsnUeH30/ssX9c4bG+mkT5skzh/YNtS+blcZS7YW88Cnq5jQtzXvL8wJ7btwUDs+WLR7u64vbh1Dx5axvDl3C/fXCbqEc+6ANqFkzo1t5fYScstquPaV+fvs9+uxXXj+m40N2v995WC+WLmTvLIavlufT3piFLtKA18SdUmL483rR5CeGH1Exi4iIiL1GWMWWmuH7LefAkfSXFlr6XTnVAD+ck5vfnFKp7D9arw+Hvt8LS99v4lV908gK6+ChGgXYx/7GoBLh7bn3nP7EBMZPrlmpdu7329mpyzbwU1vLqJtixi27SVB6ry7TqNV8M2ux+en293TAFjzwETyymoY/eisUN+HJvXlsqEdcDhM6MPIfef25ppR4Z+jiIgcuNJqD5c8P4c1O8uY2Kc195/Xh2U5JVz3+r7fT6x78MzQcuNw/vjeUt5bmLPX39c7S6oZ8cjMBu2xkU6W/GU8OUWVjHv8m/2Ov3dGImN7pPHc1xu5eHA73lsYPpg0ulsqr147jLJqD+Me/4bCit3LvP40sQc3/qTrfq91oKy1PPDZal7+YVO99s5pcQzukNxgjKvun4Dfwp8/XI7T4eCDRTn8+exeXDe6c6hPtcdHlMvB9FW7mLOxgD9O6EFclCbDi4iIHC0KHMkxr6jCzcAHZoS2687SqXL7eH9RDvd/uhKPL3APR7kcrH3wzFCfujN89jy+1tysAn72wo88d8UgzuyXUa89wuXA7fUzdfkO5m0qZM3OMubcOY7s/Eo2F1Rwx/+Wh/p/futoerZOrHful7/f1OBb4dhIJ5Xu3fmaUuMj6+VzGNW1Jfed20fVW0REDtG9H684oKqa6x48k7s/XM7ircU8e8UgurWKb5RlUSu3l/DKD9n1ZhIBjOicwo9Zu3P+RLkceHx+zj8psGTu399s5MPF23ji0pNCf088Pj8RTgd/en8p7y7Yfb5Tuqbyj58NoFVC/Zk505bv4DdvLAptz7lzHBlJMYf0POZmFVBU6aFXRgL55TU89/VGvlydG9r/57N70aN1AiM7t8TldFDp9vKr/y7kt6d1Y3CH5AYl60sqPSTFNpzJJSIiIk1HgSM55tUGdeo6u18GT182kLs+XM7b87fW2zegXRIf33xKaLuows1v3lgYeqP+0lVDOL13Oll55Xy1JpcHp6yud3zdwNKeQadaax6YSHSwLPDP/j2HuZsK+duF/fjZ0A4N+i7LKean//qhXtttZ3Rn7qbCUH6McOIinay8fyLbiqto2yKG2Rvz6doqvsEHBJGjaUNuGdERTtol7z3XmEhT8PktN7y+gJlrdgc1YiKcXD+mM+f0z2Dl9hJen7OZ1PgozurXmtHd0li9o5TR3dKO6Lhmrt6Fw2HIbBnHqX//OtTeMi6SF64awoB2SQdV9XNDbjlnPvktMRFO/vvL4WGrkkFgFm3dJXNf3TaWzmnxBzX2T5Zu57dvLQ6776FJfTm7XwYtYiPD7hcREZFjhwJHckzbWlgZWtp11ciOvF7n2+O/XdiP2z/YPdvn9ok9+dvna7j33N5cG2bpwJaCSsY8NqtB+56+/sNPyEyN4/U52fzl45UN9g9o34KPbxoV2t5VWs0L32Zx2/jue13qtmp7Kb94dT47S6sxBubddToA7y7YysrtJZRVe3E6DLee3p3ZG/N59PO1RLocPPGzk7ixzrfGwzqlMKBdEn3bJnHeSW3DXkvkSHn26w08+nmgmmB8lIvhnVL4w4Qe9EhPoKTKQ0ykMxRQhcCSlmdmbWBedhHXj+7UaB/Q/X5LpcdHvJaySB05RZWc8rf6v+On/nY0vdsk7uWIo2/m6l388rUF3D6xJ7/5SZdDPo+19oBmRX24OIffvbMUgI4tY7nltG78/t2l/PNnA5g0sF2D/n6/5epX5jGgXQv+MKEH/e/7gtJqb4N+4ZJ0i4iIyLFLgSM5ZllrGfrQl6ElXPee25u/fho+Eejcu06jVUIUi7cWM7B9i7BvqOvmSqqrZVwkD03qx1Mz17NqRyCR9aZHzgr1vfX0blw4qN0+q7k1tn9MX8tTX+27nHNttR+RI23J1mI+WbK9QU6TWilxkVS5fXj9fq4b3Zm8spoGy3MANj58Fk7HwS8Bstby3x83M3tDAaO6tuTt+VtZub2Uhyb1ZdLAtke8alRz5/dbyoIVHSvdXqJdTpLjTrxZIAs3F3Lhc3N46aohjO6eit/PXnPanUju+2Qlr87OrteWGh/Jgj+fAcDSrcVc+sKPVHl89fqkJUSRV1bDZcM68MgF/ULtxZVukmIiVOVMRETkOHKggaMT+123NEsv/5BNfrmblLhI/nZhf0Z3SyWvrIYrR3Zk5CNfhfp996dTQ5VXBnVI3uv5jDGseWAigx+YQUUwv9DjFw9gbI80UuOjOLlrS/rfNx0gFDTqlZHIrad3P1JPca/a1QlStUqI4pSuqXy1NpfiSk+o/fIX5zZIMBrO1sJKFm4uYkhmspYXyX5Za/loyTamLNvJl6t30a9tEsu3lQCBZT+L/3IGczcV4vb6Wbm9hCe+XF8vEe9zXweqJ9V+6BzXsxWndE3l/s9W0eWuqZzTP4PLh3UILa+pTYD72bLt3Pzm7iUxfdok8uZ1I0iKjeC9hTmh2X+fr9wZ6nP3hyu4+8MV3HFmT3499tBnb+yppMrD2p1lDOrQgu3F1bRLjmmQp6Uxfbg4hwXZRbi9fq4amUnftolYC/nlNaTERVLp8fHp0u2c2TeDv09fS5XbxyMX9CPS6eD37y7hoyXbG5wzNT6S03qmc/XJmRRVuunTJvG4XlJUXOlmZjDvTtvkGKJcChjVuu+nffhqTS5bCitDbfnlbr5as4vR3dK48Y1FDYJGAHllgQpnHfb40uR4vo9ERERk3zTjSJqdcX//mqz8Cr7706kNZvtsK65i1ORA8OhQStpPXb6D2z9Yxve3j6tXbnnPnBCr7p/QJLMZcsuque3dpZzWs1WoYk9uWTXDHgpU6bnp1C48MyvwAf2r28ZS6fZx0fOzuWF0Z353RneMMfj9lmXbSjj/md35lVonRnP/eX0Y36d1qK202kNljY/WScqddKL5YUM+nVLjmL5yJ58s3c4NY7rw0ndZPfUjnAAAFBdJREFULNhc1KBvXKSTj24a1SBhe1GFmydnrufCQe1IiongzXlbiHAabh7XlWqPn4QoFz5rQ9UFD1ZKXCSFFYEA8lOXDiSnqJIureLpmhbPY9PX8mawfHrWw2fhcBhyS6uZvbGAXhmJLN1azLNfbyC7oJLLhrWnXXIsQzom0yk1jlaJ0RRXuvnfom0M65RCjddPdn4Fu8qqeWLGetw+f2gMUS4HbZNj+PfPBzd6wvrc0mqGPVy/+lZitCu0PCjS6ag3llpDOiaTW1bDlsJK+rVN4rRerfD7LXFRLrILKlm0uYh1uWXU/mlPjY9ixu/GUO310ToxusFskQNd+gSBGU7GcFgzTirdXp74cj3frsujdVI0t53RgxaxEWTlV+D1+RnRuWUoqOj3W3aUVvPtujyy8sq5YnhHWidFU1LlYXtxFfM2FfL6nM1sK64iwmlY8Ocz6v1el0BustU7ykiNj2J7cRW3vbe0QZ+HJvWlU2oc/du14IsVO3nxuyzG9kjj92d0VyBORETkOKelanJMWbuzjHs/WcHZ/TK45+OVjOrakjeuGxG2b0WNl7Jqb6MHPHaUVPHXT1Zx2/juza6qmcfnx+UwGGOYt6mQS/49p0GfF64czID2LRj+cMNS0LWuOTmT+37ah9kb8rn8pbkATLtlNL0ymk8uEGk8uWXVfL5iJ16fZeX2UmIjnSzeWsSKbaVh+w/q0II3rx+By2HILauhTYtDq8ZUV0mVhxqvjyq3jzs+WE5RpZs1O8tC+2MjnXx/+zicDoMx8NJ3m/hwcQ4RDgcVbi8vXjWE/u0aJgF+ZtYGHvsikHdpb0GWcCYNbMvcrAK2l1Tvs5/TYfD5bWiMf5zQg4ToCLYUVtK2RTQntU8mKSaCJVuLSY6NoG/bJKIjnOSWVVNc6WHFthK2F1eTX15Dv3ZJOI2hVWIUHVPieGbWBt5ZEEju/+iF/fH4/ewsqWb5thI25VfgchginA6SYyOZ0CedrPwKiis9fLI0MMMoMdrFRYPbc885vcIGcVbvKGV+diFzNxUyZdmOevt+0iONpJgI2rSIweP18+6CrcRHuejdJpGx3dNo0yKGIR1T+H5DPut2lXHhoHbkldfwv0U5fLp0O+1TYrltfHdS46PonZG43+TOtYGpz1fs4A/vLaM8uLQuOTaCojozKeuKjXTSr20SczcVht1fV/uUGCb0bs2lw9rTtVXz+r3dHA3463RKqnb/3JfeO17BNhERkROYAkdyTPnVfxfwxcpdoe0nLz1JSaD3orjSzUn3z6jXFu5DWO3PcOnWYh6ZtjpUXe7la4bwi1d3v6ZGdm7Ji1cPIS7SqdwVxziPz8+aHWV8snQbU5fvZFtxVYM+xkCn1Diy8ioAePdXI8kvryEjKZo+bZKIdB14lafDYa1lQ245XdLiD2k5mNvr5+/T1/LVmlw25JYDcPXIjhgTCED98pROoYBSjdfPVf+ZF/p51Fbcqqzx0qVVPMtyShjWKRmvz3L+wLZEBIMhFTVefvna/Hol1BuDy2EY0bkl3dMTuOPMngf8M1+9oxSPzx82kBaO3295+qsNrM8twwIGWLm9lLyymlAA52C0iI2gyu2jxhsI0tUGor0+PyVVHuZtKuTRL9YS6XSQU1SJw2EoC86gchiIdDm4fFhHOqXGcvnwjhRXuvls2Q52llbTKTWO2Egns9bksaOkipIqDyu3l3LNyZlcMKgtXr/l2Vkb8fr9ZLaMo2PLWE7pmtrsgvzNnc9vGfPoLNq0iOat60ccVFU3EREROf4ocCTHjLJqD/2COYYAJl/Qj0uHNSxvL7t9sy6P3hmJzFqbS/f0BD5dup3/fB9IYHzx4HY8dvGABsdk5ZUz7vFv6rVdMLAt/1u8LbR9Vr/WjO/dmvMHKmh3NFgbmAmUFBNxUEnY/X7Lc99s5LEv1hLlcjCwQws6pMTy3sIc6v5K79gylutGd6bK7SUlLpAzq1VCFA6HodLtJbe0hszUuCPwzI4ev99S7vaSGL3vWRPWWr5em4fFcmqPVge1PMvrtxRU1LApr4IB7VuwKb+CrPwKckurSUuIItLpYNbaXBZuLuInPVrRp00inVLj6JwWjzGQU1hFTKSTVdtLWbi5iEuHtad7Ewc8aryBAFBidATVHh8RTgfrdpWxIbecHzbks2BzEbee3o0dxdVERziY0Kc1rRKjKa/x8vzXG/nXrEAS/2GZKWTlV5BfXhM6tzHUuw8n9EknOsLJpUM7MLJLy6P9VGUPbq8fn98qgbiIiIgocCRH3o9ZBdz85iKuGpnJW/O2cEbvdC4f3oG/fLyS0ioPnVLjePySAaFcQVl55bRpEUN0hJOiCjfPzNrA7I0FoYpmPx/RgQfP77evS8pe+PyWwgo3qfGB5KV7+1B805uLQktX1j44kWq3n3P+9R1bC+vPTPnLOb35xSmBHEs/bMjn5jcX8fglAxjXM/0IPovmZXNBBdkFlSTFRNCzdQIOY6jy+A55WUdJlYc1O0opq/bis5a8shr+/NGK0P7MlrFERzhJS4giIymaJVuLQ68dp8MQHeEg2uVke0k1Wwsr9zpjpENKLC9fM5SureIPaZwiB+L1Odm8+F0Wbq8fl8PB8E4p9GmbxJl9W9MyPhJrAzmiNItRREREpPlS4EiOqE+Xbuf/3lq83369MhJ58Py+vDN/C+8uCJTpbpUQRaXbV++D77DMFN6+YcQRrWAkgVkXk6etYXjnlHpBoLU7yyit9nDfJytZuT18/huAs/tncNGgdozulhp2icOSrcU8+eU6kmICuWD6tEmifUoMz8zaSJTLwR8n9GDSwLaU13h5e/5Wlm8rYWNuOTee2pWfDmhzRJ7zFyt3khQTwaAOyRRXulmaU8Lb87bg9vnplZFIm6RoBndMITHGxRtzt/Dd+nzio5zMz26YKBoC+XTO7NeaaJeTNi1iGJqZTKTLQbXHz6ItRcRGOtlZEkjUXFrtIaeoitaJgYS+4SoYjemeRtsW0WwvrqbS7Q1dt2frBLx+S1ykk0iXg/nZRcREOBnROYWOLeMY1DGZc/tnYC14/H62F1djgPYpsTj1OhIRERERkf1otoEjY8xE4EnACbxkrZ28r/4KHB19tQlhHWGq5xRWuPn9u0v4em0eAEMzk2kRG8m95/bms2U72FpYyXWjO9M6MZpz//V9KPfInkZ0TuHKEZlM6JOuHAvNzILsQi56fnfy7Ql90slIiuHV2dmhtoRoF2XVXmIinJw/sA2Vbh9xUa5QpSsI5NHZUlgZup8SolyU7SOvyojOKfRsnUj7lFhKqzwMzUyhZ0YCqfFRQGDJ0JKcYqo9PkZ2bokxhoWbC/nP95uYvbGAtPgoWidFkxgTQWmVh2qPj1XbS6lwNwzW7E+75BhiIpxcPrwDCdERzN9UyKaCwHKc2txA+9O2RQzxUS7W7iojIymaU7qmMjQzhZbxkWzKryAlLpIzeqeTsMcSq4WbC4mNdDVIWF7l9hHhNHq9iIiIiIhIo2iWgSNjjBNYB5wB5ADzgcustav2dowCR1BQXsPO0mpyiqqICy5dqfH6yMqroLzGS3piNMmxEURFBJYFlFd7cTkMXr+ltNpDy7go3D4/ZdUeyqq9of9Lqjy4vX7Ka7yU13ipqPGSW1ZDldtHlMuB30K1x4fDYUiIchEb5aSw3E2F20fn1Dje/fXI0If6cDYXVDBj1S6+W5+P31pevGoIPr8NjVmaL6/Pz4rtpSTHRtCxZSAHzo6SKr5ctYvNBZVMXb4jVJXKYcBf59fIa78YxtjuaQBsyC2ntNrDoA7JWGv5YuVOnv16Iz3SEzirfwYjO7dk8ZZiHp++Fp+1LMspCQWaarVKiKJzWhzZ+ZXsLA1cMybCSXy0i7yy3XlV0hOjqPH6Ka70EB3hICE6gryyGtISoji9VyuWbythbPc0MlvGcUbvdLx+y7qdZcRFuVizs5QdJdV0a5XAkMzkfd6fO0qqWLS5mLE90thaWMmHi7eRFBNBp9Q4+rVNwue3lFR56JYeH1pqJiIiIiIi0tw018DRSOA+a+2E4PadANbaR/Z2zPEQOPpgYQ7/+X4TKXGR+Py23gfjGp+fXSXVGENoX21FH6/fj9vrp7T64Kvf7IvDQEJ0BIkxLqJcTuKjXCREu4iNdJIQHUFSTARl1R4cxtAiNpJKtxef31Lp9hEd4eDiIe0Z1CG5UcckxxZrLW6fnyiXE7fXjzHw4eJttEmK4ZRuqYd83vIaL5VuL/llbtbuKmXOxgKqPX5W7Sila1o83dPjWbC5CKfDEB/lwuPzM2lgOyb2bR1anlVS6SExJhCwWbm9lF4ZiVq6JSIiIiIisocDDRwd7a/D2wJb62znAMP37GSMuQG4AaBDh2O/ulZCtIv0xChKqjw4HQanw2AIfJBNcLnoHvygXbvP4wuUOnY5HbgchoykGDJbxpKWEJhR4XIYoiOc1Hj9dEmLw+0LzLJwe/14/ZaEaBdenyXCaYiPdpFf5g7NwKgNEClhqRwOYwxRrkBFntpS4pcMaX/Y542PchEf5aJVQjS92yQyaWC7gz5HUuzupV992yYd9phEREREREROZM1yHYW19gXgBQjMOGri4Ry28X1aM75P6yN6jYykmEPaJyIiIiIiIiKyN0c7y+o2oO60hHbBNhERERERERERaWaOduBoPtDNGNPJGBMJXAp8cpTHICIiIiIiIiIiB+CoLlWz1nqNMTcDXwBO4GVr7cqjOQYRERERERERETkwRz3HkbV2KjD1aF9XREREREREREQOztFeqiYiIiIiIiIiIscIBY5ERERERERERCQsBY5ERERERERERCQsBY5ERERERERERCQsBY5ERERERERERCQsBY5ERERERERERCQsBY5ERERERERERCQsBY5ERERERERERCQsBY5ERERERERERCQsBY5ERERERERERCQsBY5ERERERERERCQsBY5ERERERERERCQsBY5ERERERERERCQsBY5ERERERERERCQsY61t6jHskzEmD9jc1ONoBKlAflMPQuQo030vJxrd83Ki0T0vJxrd83Ii0n1//OporU3bX6dmHzg6XhhjFlhrhzT1OESOJt33cqLRPS8nGt3zcqLRPS8nIt33oqVqIiIiIiIiIiISlgJHIiIiIiIiIiISlgJHR88LTT0AkSag+15ONLrn5USje15ONLrn5USk+/4EpxxHIiIiIiIiIiISlmYciYiIiIiIiIhIWAocHQXGmInGmLXGmA3GmDuaejwih8oY87IxJtcYs6JOW4oxZoYxZn3w/+RguzHGPBW875cZYwbVOebqYP/1xpirm+K5iBwIY0x7Y8wsY8wqY8xKY8wtwXbd93JcMsZEG2PmGWOWBu/5vwbbOxlj5gbv7XeMMZHB9qjg9obg/sw657oz2L7WGDOhaZ6RyIExxjiNMYuNMZ8Ft3XPy3HNGJNtjFlujFlijFkQbNP7GwlLgaMjzBjjBJ4BzgR6A5cZY3o37ahEDtmrwMQ92u4AZlpruwEzg9sQuOe7Bf/dADwHgT9IwL3AcGAYcG/tHyWRZsgL3Gat7Q2MAG4K/g7XfS/HqxpgnLV2AHASMNEYMwL4G/BPa21XoAj4ZbD/L4GiYPs/g/0Ivk4uBfoQ+LvxbPA9kUhzdQuwus627nk5EZxqrT3JWjskuK33NxKWAkdH3jBgg7U2y1rrBt4GzmviMYkcEmvtt0DhHs3nAa8FH78GnF+n/XUb8CPQwhiTAUwAZlhrC621RcAMGgajRJoFa+0Oa+2i4OMyAh8q2qL7Xo5TwXu3PLgZEfxngXHA+8H2Pe/52tfC+8BpxhgTbH/bWltjrd0EbCDwnkik2THGtAPOBl4Kbht0z8uJSe9vJCwFjo68tsDWOts5wTaR40W6tXZH8PFOID34eG/3vl4TckwKLkcYCMxF970cx4JLdpYAuQQ+BGwEiq213mCXuvdv6N4O7i8BWqJ7Xo4tTwB/AvzB7ZbonpfjnwWmG2MWGmNuCLbp/Y2E5WrqAYjI8cNaa40xKtUoxx1jTDzwAXCrtbY08OVygO57Od5Ya33AScaYFsCHQM8mHpLIEWOMOQfItdYuNMb8pKnHI3IUnWKt3WaMaQXMMMasqbtT72+kLs04OvK2Ae3rbLcLtokcL3YFp6oS/D832L63e1+vCTmmGGMiCASN3rDW/i/YrPtejnvW2mJgFjCSwLKE2i8c696/oXs7uD8JKED3vBw7RgE/NcZkE0gpMQ54Et3zcpyz1m4L/p9L4EuCYej9jeyFAkdH3nygW7AyQySBpHmfNPGYRBrTJ0BtBYWrgY/rtF8VrMIwAigJTn39AhhvjEkOJs8bH2wTaXaCeSv+A6y21v6jzi7d93JcMsakBWcaYYyJAc4gkNtrFnBRsNue93zta+Ei4CtrrQ22XxqsQNWJQELVeUfnWYgcOGvtndbadtbaTALv07+y1l6B7nk5jhlj4owxCbWPCbwvWYHe38heaKnaEWat9RpjbibwAnICL1trVzbxsEQOiTHmLeAnQKoxJodAFYXJwLvGmF8Cm4FLgt2nAmcRSA5ZCVwLYK0tNMY8QCCoCnC/tXbPhNsizcUo4EpgeTDnC8Bd6L6X41cG8FqwGpQDeNda+5kxZhXwtjHmQWAxgYAqwf//a4zZQKB4wqUA1tqVxph3gVUEqhPeFFwCJ3KsuB3d83L8Sgc+DC69dwFvWms/N8bMR+9vJAwTCJCLiIiIiIiIiIjUp6VqIiIiIiIiIiISlgJHIiIiIiIiIiISlgJHIiIiIiIiIiISlgJHIiIiIiIiIiISlgJHIiIiIiIiIiISlgJHIiIiIiIiIiISlgJHIiIiIiIiIiISlgJHIiIiIiIiIiIS1v8HIP2hpSzsf+MAAAAASUVORK5CYII=\n",
      "text/plain": [
       "<Figure size 1440x576 with 1 Axes>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "plt.figure(figsize = (20, 8))\n",
    "plt.plot(data)\n",
    "plt.title(stock_name + '- Amazon', fontsize = 18)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Q-Learning with Replay Buffer"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "The Algorithm we are implementing here is the following:\n",
    "\n",
    "![title](./blog_stuff/dqn_classic.png)\n",
    "\n",
    "For more details read the blog [here]()."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [],
   "source": [
    "class AgentDQN():\n",
    "    # init functions\n",
    "    def __init__(self, input_dim, scope, is_eval = False, epsilon_decay_steps = 1000):\n",
    "        # input_dim: state size\n",
    "        # is_eval: is being evaluated\n",
    "        # scope: scope of the model\n",
    "        self.state_size = input_dim\n",
    "        self.action_space = 3 # sell, sit, buy\n",
    "        self.memory = deque(maxlen = 10000)\n",
    "        self.inventory = [] # holdings that we have\n",
    "        self.scope = scope # name of scope\n",
    "        self.is_eval = is_eval # whether in training or deployment\n",
    "        self.gamma = 0.95 # discount factor\n",
    "\n",
    "        # greedy-epsilon policy\n",
    "        self.epsilon = 1.0\n",
    "        self.epsilon_end = 0.01\n",
    "        self.epsilon_decay_val = (self.epsilon - self.epsilon_end)/epsilon_decay_steps\n",
    "\n",
    "        # code to load previously trained model --> skip for now\n",
    "        # self.model = load_model(\"models/\" + model_name) if is_eval else self._model()\n",
    "        self._build_model()\n",
    "        self._build_loss()\n",
    "        self.initialize_network()\n",
    "\n",
    "    # inhouse functions\n",
    "    def _build_model(self):\n",
    "        self.input_placeholder = tf.placeholder(tf.float32, [None, self.state_size], name = 'inputs')\n",
    "        self.target_placeholder = tf.placeholder(tf.float32, [None, self.action_space], name = 'target_value')\n",
    "\n",
    "        # layers\n",
    "        h1 = tf.contrib.layers.fully_connected(self.input_placeholder, 64)\n",
    "        h2 = tf.contrib.layers.fully_connected(h1, 32)\n",
    "        h3 = tf.contrib.layers.fully_connected(h2, 8)\n",
    "        self.action_pred = tf.contrib.layers.fully_connected(h3, self.action_space)\n",
    "\n",
    "        # print the values\n",
    "        print('inputs:', self.input_placeholder)\n",
    "        print('target_value:', self.target_placeholder)\n",
    "        print('action_pred:', self.action_pred)\n",
    "\n",
    "    def _build_loss(self):\n",
    "        self.loss = tf.reduce_mean(tf.square(self.target_placeholder - self.action_pred))\n",
    "        print('loss:', self.loss)\n",
    "        self.update_step = tf.train.AdamOptimizer(0.001).minimize(self.loss)\n",
    "\n",
    "    # user callable functions\n",
    "    def initialize_network(self):\n",
    "        self.sess = tf.Session()\n",
    "        self.sess.run(tf.global_variables_initializer())\n",
    "\n",
    "    def save_model(self, frozen = False):\n",
    "        # code to save the model as frozen inference graph (optional) or \n",
    "        # checkpoint file (default)\n",
    "        if frozen:\n",
    "            # convert to frozen graph here\n",
    "            model_path = 'some_path'\n",
    "            print('Model saved at {0}'.format(some_path))\n",
    "            return\n",
    "        return\n",
    "\n",
    "    # operation functions\n",
    "    def act(self, state):\n",
    "        if not self.is_eval and np.random.random() <= self.epsilon:\n",
    "            return np.random.randint(self.action_space)\n",
    "\n",
    "        # else use the model to predict action\n",
    "        action_dist = self.sess.run(self.action_pred, feed_dict = {self.input_placeholder: state})\n",
    "        return np.argmax(action_dist[0])\n",
    "\n",
    "    def experience_replay(self, batch_size):\n",
    "        # make the minibatch\n",
    "        mini_batch = []\n",
    "        mem_len = len(self.memory)\n",
    "        for i in range(mem_len - batch_size + 1, mem_len):\n",
    "            mini_batch.append(self.memory[i])\n",
    "\n",
    "        # loss log\n",
    "        loss_log = []\n",
    "\n",
    "        for state, action, reward, next_state, done in mini_batch:\n",
    "            target_s = reward\n",
    "            if not done:\n",
    "                # get predictions from model\n",
    "                pred = self.sess.run(self.action_pred, feed_dict = {self.input_placeholder: next_state})\n",
    "\n",
    "                # get the target value to be fit\n",
    "                target_s = reward + self.gamma*np.amax(pred[0])\n",
    "\n",
    "            # target value to be fit upon\n",
    "            target_y = self.sess.run(self.action_pred, feed_dict = {self.input_placeholder: state})\n",
    "            target_y[0][action] = target_s\n",
    "\n",
    "            # train the model\n",
    "            feed_dict = {self.input_placeholder: state, self.target_placeholder: target_y}\n",
    "            loss, _ = self.sess.run([self.loss, self.update_step], feed_dict = feed_dict)\n",
    "\n",
    "            # add to logs\n",
    "            loss_log.append(loss)\n",
    "\n",
    "        # reduce the value of epsilon\n",
    "        if self.epsilon > self.epsilon_end:\n",
    "            self.epsilon -= self.epsilon_decay_val\n",
    "\n",
    "        # return loss\n",
    "        return loss_log"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "inputs: Tensor(\"inputs:0\", shape=(?, 100), dtype=float32)\n",
      "target_value: Tensor(\"target_value:0\", shape=(?, 3), dtype=float32)\n",
      "action_pred: Tensor(\"fully_connected_3/Relu:0\", shape=(?, 3), dtype=float32)\n",
      "loss: Tensor(\"Mean:0\", shape=(), dtype=float32)\n"
     ]
    }
   ],
   "source": [
    "# define the agent\n",
    "agent = AgentDQN(window_size, 'model_pre')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 5360/5360 [05:14<00:00, 17.02it/s]\n",
      "  0%|          | 2/5360 [00:00<05:04, 17.61it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[*]Episode: 0, loss: 0.0945732370018959, profits: 36.03064699999961\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 5360/5360 [05:18<00:00, 16.84it/s]\n",
      "  0%|          | 2/5360 [00:00<05:22, 16.59it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[*]Episode: 1, loss: 0.0045433491468429565, profits: 9.690002\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 5360/5360 [05:16<00:00, 16.92it/s]\n",
      "  0%|          | 2/5360 [00:00<05:33, 16.04it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[*]Episode: 2, loss: 0.004467894323170185, profits: -4.844200000000008\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 5360/5360 [05:42<00:00, 15.65it/s]\n",
      "  0%|          | 2/5360 [00:00<06:35, 13.53it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[*]Episode: 3, loss: 0.0053507015109062195, profits: -10.769626000000136\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 5360/5360 [05:35<00:00, 15.99it/s]\n",
      "  0%|          | 2/5360 [00:00<04:55, 18.14it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[*]Episode: 4, loss: 0.009466061368584633, profits: 7.665975000000092\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 5360/5360 [05:42<00:00, 15.63it/s]\n",
      "  0%|          | 2/5360 [00:00<05:41, 15.70it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[*]Episode: 5, loss: 0.009681923314929008, profits: 20.574966999999784\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 5360/5360 [05:56<00:00, 15.05it/s]\n",
      "  0%|          | 2/5360 [00:00<06:11, 14.42it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[*]Episode: 6, loss: 0.0054457238875329494, profits: -15.291315000000065\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 5360/5360 [05:08<00:00, 17.36it/s]\n",
      "  0%|          | 2/5360 [00:00<04:57, 18.02it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[*]Episode: 7, loss: 0.21783007681369781, profits: 39.67718099999969\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 5360/5360 [05:00<00:00, 17.87it/s]\n",
      "  0%|          | 2/5360 [00:00<05:01, 17.80it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[*]Episode: 8, loss: 0.00534070236608386, profits: -2.4111870000000657\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 5360/5360 [05:22<00:00, 16.62it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[*]Episode: 9, loss: 0.0012952237157151103, profits: -34.3263249999998\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n"
     ]
    }
   ],
   "source": [
    "# run things\n",
    "for e in range(episode_count):\n",
    "    state = get_state(data, 0, window_size + 1)\n",
    "\n",
    "    # init values for new episode\n",
    "    total_profit = 0.0 # total profit in this epoch\n",
    "    agent.inventory = [] # reset the inventory\n",
    "    total_loss = [] # at each step what was the total loss\n",
    "    mean_loss = [] # at each step what was the mean loss\n",
    "\n",
    "    for t in tqdm(range(len_data)):\n",
    "        action = agent.act(state)\n",
    "\n",
    "        # next state\n",
    "        next_state = get_state(data, t + 1, window_size + 1)\n",
    "        reward = 0\n",
    "\n",
    "        # now go according to actions\n",
    "        if action == 2:\n",
    "            # buy\n",
    "            agent.inventory.append(data[t])\n",
    "            if LOG:\n",
    "                print('Buy:' + format_price(data[t]))\n",
    "\n",
    "\n",
    "        elif action == 0 and len(agent.inventory) > 0:\n",
    "            bought_price = agent.inventory.pop(0) # remove the first element and return the value\n",
    "            profit = data[t] - bought_price # profit this iteration\n",
    "            reward = max(data[t] - bought_price, 0) # reward\n",
    "            total_profit += profit # add to total profit\n",
    "            if LOG:\n",
    "                print(\"Sell: \" + format_price(data[t]) + \" | Profit: \" + format_price(profit))\n",
    "\n",
    "        # condition for finish\n",
    "        done = t == len_data - 1\n",
    "        agent.memory.append((state, action, reward, next_state, done))\n",
    "        state = next_state\n",
    "\n",
    "        if done and LOG:\n",
    "            print(\"Total Profit: \" + format_price(total_profit))\n",
    "\n",
    "        # train the model\n",
    "        if len(agent.memory) > batch_size:\n",
    "            losses = agent.experience_replay(batch_size)\n",
    "            total_loss.append(np.sum(losses))\n",
    "            mean_loss.append(np.mean(losses))\n",
    "\n",
    "    # add the mean loss to global loss\n",
    "    loss_global.append(np.mean(mean_loss))\n",
    "    profits_global.append(total_profit)\n",
    "    \n",
    "    print('[*]Episode: {0}, loss: {1}, profits: {2}'.format(e, loss_global[-1], profits_global[-1]))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "[<matplotlib.lines.Line2D at 0x1210c1390>]"
      ]
     },
     "execution_count": 9,
     "metadata": {},
     "output_type": "execute_result"
    },
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAXwAAAEICAYAAABcVE8dAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADl0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uIDIuMi4yLCBodHRwOi8vbWF0cGxvdGxpYi5vcmcvhp/UCwAAIABJREFUeJzt3Xd81fX1+PHXyU7IYiSBLFYwhD0ioDiAYHFDrVpo66621n6/dtli++u37be12tL5/Vpt7VC/LkRLwdEWFXArkLD3TAhhBZIQRsg8vz/uDQZMQsa993PHeT4eeZB87ufez+FCTj55j3NEVTHGGBP8wpwOwBhjjG9YwjfGmBBhCd8YY0KEJXxjjAkRlvCNMSZEWMI3xpgQYQnf+A1xeVJEKkVkpYhcKiLbnI7LH4jICREZ5OlzTWixhG+6RUSKRaTGnWQOichTIhLfxZe7BLgCyFTVCar6nqrmnnOt6R4J3EdEZIqI7Ovu66hqvKru9vS5JrRYwjeecJ2qxgPjgHzg/517gvvu/Xz/3/oDxap60gsxep2IRPjyecZ0liV84zGqWgb8CxgBICJvi8hDIvIBcAoYJCLpIvKKiFSIyE4Rudt97l3AX4CL3L8t/KTl3bGIPANkA6+6H/+uiMSIyLMiclREqkRklYiktRabiOS546kSkU0icr37+EQROSgi4S3O/ayIrHd/HiYic0Vkl/s6C0Skl/uxASKiInKXiOwFlp1zzR7u9yPdHfMJ99//xyLysjv2auB2EZkgIh+54zsgIo+KSFSL11IRyXF//pSI/EFEXheR4yKyQkQGd/Hcz4jINhE5JiKPicg7IvLlzv/rm0BgCd94jIhkAVcDa1ocvgW4B0gASoD5wD4gHbgR+LmITFPVvwJfBT5yD0n8qOVrq+otwF7cv02o6i+B24AkIAvo7X5+TStxRQKvAm8AqcB/AM+JSK6qrgBOAtNaPOULwPPuz/8DmAVc7o65EvjDOZe4HMgDZpwT80ngKmC/O+Z4Vd3vfngm8DKQDDwHNALfBPoAFwEFwNfO/bu0MBv4CdAT2Ak81NlzRaSPO4YHcb1/24CL23kdE+As4RtPWCQiVcD7wDvAz1s89pSqblLVBqAvMBn4nqqeVtW1uO7qb+3idetxJaocVW1U1SJVrW7lvElAPPCIqtap6jLgNWCO+/EXmj8XkQRcP7RecD/2VeAHqrpPVWuBHwM3njMM82NVPamqn/ph046PVHWRqjapao079o9VtUFVi4E/4fpB0pZ/qOpK9/v6HDCmC+deDWxS1YXux/4HONiJv4MJMDZ2aDxhlqq+1cZjpS0+TwcqVPV4i2MluMb9u+IZXHf380UkGXgWV3KuP+e8dKBUVZvOuW6G+/PngQ9F5F7gBmC1qpa4H+sP/ENEWj63EWg5dNTy79hRZz1HRC4AfoPrvYjD9b1Z1M7zWybmU7h+oHX23PSWcaiqemKC2fgvu8M33tayHOt+oJf7LrpZNlDWhddCVetV9SeqOgzXUMS1tP7bwn4g65xJ4zPXVdXNuH4AXMXZwzngSohXqWpyi48Y93xFq3G1F3M7xx8HtgJDVDUR+D4g7byuJxwAMpu/EBFp+bUJPpbwjc+oainwIfCwe8J1FHAXrjvzjjgEnFlfLiJTRWSke8K1GtcQT1Mrz1uB6872uyISKSJTgOtwzSc0ex64H7gMeKnF8T8CD4lIf/c1U0RkZgfjbY65t4gknee8BPff4YSIDAXu7cQ1uup1YKSIzHIPUd2Ha9jNBClL+MbX5gADcN11/wP4UTvDQed6GPh/7pUs38GVnF7GlSi34Jo/eObcJ6lqHa4EfxVwBHgMuFVVt7Y47QVcY+bLVPVIi+O/B14B3hCR48DHwMQOxov7Gi8Au91xp7dx6ndw/XZxHPgz8GJHr9FV7r/nTcAvgaPAMKAQqPX2tY0zxBqgGGPAtQQV1wqqL6rqcqfjMZ5nd/jGhDARmSEiySISzSfzBh87HJbxEkv4xoS2i4BduIa6rsO14qozy0tNALEhHWOMCRF2h2+MMSHCrzZe9enTRwcMGOB0GMYYE1CKioqOqGrK+c7zq4Q/YMAACgsLnQ7DGGMCioiUnP8sG9IxxpiQYQnfGGNChCV8Y4wJEZbwjTEmRHgs4YtIuIisEZHX3F8PdHfX2SkiL7bs3mOMMcb3PHmHfz+uAlbNfgH8VlVzcHUJusuD1zLGBKhFa8qY/MgyBs59ncmPLGPRmo5Wxzbd5ZGELyKZwDW4uhc119WehquSIcDTuNrEGWNC2KI1ZTy4cANlVTUoUFZVw4MLN1jS9xFP3eH/Dvgun9Qi7w1UudumgasCX0ZrTxSRe0SkUEQKy8vLPRSOMcYfzVuyjZr6xrOO1dQ3Mm/JNociCi3dTvgici1wWFXba8fWJlV9QlXzVTU/JeW8G8WMMQFsf1XrddnaOm48yxM7bScD14vI1UAMkIiraUSyiES47/Iz6XgbO2NMkEpPjqWsleSenhzrQDShp9t3+Kr6oKpmquoAYDaujkFfBJYDN7pPuw1Y3N1rGWMC2wMzcomOODvtxEaG88CMXIciCi3eXIf/PeBbIrIT15j+X714LWNMAJg1NoPPjvtkOi9M4OezRjBrbKtTfMbDPJrwVfVtVb3W/fluVZ2gqjmqepOqWp9MYwzaBEmxkfzm5tE0KQxKjXc6pJBhO22NMT5VWFLB+P49mZqbSpjA0i2HnA4pZFjCN8b4TOXJOnaVn2R8/5707BFFfv9evLXlsNNhhYyAT/i2a8+YwFFUUgnA+P49ASjIS2XzgWpblukjAZ3wbdeeMYGlsKSSiDBhdGYyAAV5aQAs3Wp3+b4Q0Anfdu0ZE1iKSioYnpFEbFQ4AINTejCgd5yN4/tIQCd827VnTOCobWhk3b5j5LuHcwBEhIK8ND7cdZRTdQ3tPNt4QkAn/LZ259muPWP8z8ayauoams5K+OAax69raOK9HUcciix0BHTCf2BGLrGR4WcdiwgT27VnjB8qKqkAYPyAsxP+hQN6kRATYcM6PuCJWjqOad6dN2/JNvZX1RAVEUZkuHDliL4OR2aMOVdhcSXZveJITYg563hkeBhTclNZtrWcpiYlLEwcijD4BfQdPriS/gdzp7HnkWt4+s4JnKht5MVVpU6HZYxpQVUpKqn81HBOs+l5qRw5Ucu6fVU+jiy0BHzCb2niwF7k9+/Jn97ZRV1D0/mfYIzxieKjpzh6su5TwznNplyQSniYsNQ2YXlVUCV8EeG+aTnsP3ba1uIb40eaN1zl9+/V6uNJcZHk9+/JWzaO71VBlfABplyQwoiMRB5/ZxeNTep0OMYYXBO2iTERDGmnUNr0vDS2HjzOvspTPowstARdwhcR7puSw54jJ3l9wwGnwzHG4JqwHde/Z7sTsgV5qQAss123XhN0CR9gxvC+5KTG89jynTTZXb4xjqo6VceOwyfanLBtNiglnkF9elgxNS8KyoQfFiZ8bcpgth48bjU6jHHY6r3NBdNaH79vqSAvlY93HeVEre269YagTPgA149OJ6tXLI8u34mq3eUb45TCYlfBtDFZyec9tyAvjbrGJt7bXu6DyEJP0Cb8iPAwvnr5YNaVVvHBzqNOh2NMyCosqWR4euKZgmntye/fk6TYSBvW8ZJuJ3wRiRGRlSKyTkQ2ichP3McHisgKEdkpIi+KSFT3w+2cG8dnkpYYzaPLd/j60sYYoK6hiXWlVR0azgHXjdqU3BSWbztsq+y8wBN3+LXANFUdDYwBrhSRScAvgN+qag5QCdzlgWt1SnREOHdfOoiPd1ecqeNhjPGdTfuPUdvQRH4bG65aU5CXRsXJOtaWVnoxstDU7YSvLifcX0a6PxSYBrzsPv40MKu71+qKL0zMplePKB5dttOJyxsT0j7ZcNXxhH/5BSlEhIkN63iBR8bwRSRcRNYCh4E3gV1Alao2T7XvAzLaeO49IlIoIoXl5Z6fqImLiuDOyQNYvq2cjWXHPP76xpi2FRZXktUrltTEmPOf7JYUG8mFA3pZ9Uwv8EjCV9VGVR0DZAITgKGdeO4TqpqvqvkpKSmeCOdTbrloAAnRETz2tt3lG+MrqkphSWWb5RTaU5CXyvZDJyitsF23nuTRVTqqWgUsBy4CkkWkufxyJuBYcZuk2Ehuvbg//9p4kJ2HjzsVhjEhZW/FKY6cqD3TsLwzprt73VptHc/yxCqdFBFJdn8eC1wBbMGV+G90n3YbsLi71+qOOycPJCYinMfe3uVkGMaEjMJi9/h9JyZsmw3o04PBKT2seqaHeeIOvx+wXETWA6uAN1X1NeB7wLdEZCfQG/irB67VZb3jo5kzIZvFa/fbr4nG+EBhSSUJMRFckJrQpedPz0tjxZ6jHD9d7+HIQpcnVumsV9WxqjpKVUeo6n+7j+9W1QmqmqOqN6lqbffD7Z57LhtEuAh/fMfu8o3xtqKSCsZlt18wrT0FeWnUNyrvbrdet54StDttW9M3KYYb8zN5qXAfh6pPOx2OMUHr2Kl6th86f8G09ozLTiY5LtJW63hQSCV8gHsvH0yjKk+8u9vpUIwJWmcKpnVh/L5ZRHgYU3NTbdetB4Vcws/qFcfM0ek8v2IvFSfrnA7HmKBUWFJBeAcLprWnIC+VylP1Z36AmO4JuYQP8LWpgznd0Mjf3t/jdCjGBKXC4kqG9UskLiri/Ce347Izu25tWMcTQjLh56QmcOXwvjz9UTHVtgLAGI+qb2xi3b6qLq2/P1diTCQTB/Wy5ZkeEpIJH+C+qTkcP93AMx+VOB2KMUFl0/5qTtd3rmBaewqGprHz8AlKjp70yOuFspBN+CMykpiSm8Jf39/DqTrrrmOMpxQWuyrTdqWkQms+2XVrd/ndFbIJH+DrU3OoOFnHCytLnQ7FmKBRVFJJRnIsfZM6XjCtPdm94xiSGm/LMz0gpBN+/oBeTBzYiyfe3UVtQ6PT4RgT8M4UTPPQcE6zgrw0Vu6psDm3bgrphA/w9Wk5HKqu5e9FjtV2MyZolFbUUH68tlsbrlozPS+VhiblnW3W67Y7Qj7hX5LTh9GZSfzxnV00NDY5HY4xAa3Q3Vmuoy0NO2psdk969YiyYZ1uCvmELyLcNzWHvRWneHX9fqfDMSagFZVUkhAdQW7frhVMa0t4mLh73ZbbjVk3hHzCB9cqgNy0BB5bvosm28JtTJcVlVQyJjuZ8C4WTGvP9Lw0jtXUn2mbaDrPEj4QFiZ8bepgdhw+wRubDzodjjEB6VhNPdsOHffYcsxzXTqkD5HhwtKttjyzqyzhu107Kp0BveN4dPlOVO0u35jOWrO3EtWuNTzpiISYSCYN6m1lFrrBEr5beJhw75TBbCyr5p3tthLAmM4qKqn0SMG09hQMTWV3+Un2HLFdt11hCb+Fz47NJD0phj8st2bnxrMWrSlj8iPLGDj3dSY/soxFa4JvGXBhcSV5/RLoEd29gmntKXDvurXVOl1jCb+FqIgw7rlsEKuKK1mx+6jT4ZggsWhNGQ8u3EBZVQ0KlFXV8ODCDUGV9Osbm1hbWuW18ftmWb3iyE1LsGGdLvJEE/MsEVkuIptFZJOI3O8+3ktE3hSRHe4/vTOw52GzJ2TTJz6KR+0u33jIvCXbqKk/eyd3TX0j85Zscygiz9tyoJqa+kaPVMg8n4K8VFYVV3LslO267SxP3OE3AN9W1WHAJOA+ERkGzAWWquoQYKn7a78XExnOXZcM4r0dR1hXWuV0OCYI7K+q6dTxQFRY7Foq6a0J25YK8tJobFLe3m6rdTrLE03MD6jqavfnx4EtQAYwE3jafdrTwKzuXstXvjQpm8SYCBvLN912ur6RqIjWv83Sk2N9HI33NBdM65fk/b/TmKxkeveIshr5XeDRMXwRGQCMBVYAaap6wP3QQSCtjefcIyKFIlJYXu4fq2MSYiK5ffJA3th8iG0HjzsdjglQtQ2NfPXZImobmogMP3sjUmxkOA/MyHUoMs9yFUyr8MlwDrhW1E0dmsrb2w5Tb7tuO8VjCV9E4oG/A99Q1eqWj6lrYXuri9tV9QlVzVfV/JSUFE+F0213XDyAuKhwHnvb7vJN59U1NHHfc6t5e1s5D98wknk3jiY1IRqApNgIHr5hJLPGZjgcpWfsq6zhUHWtT4Zzmk3PS6X6dMOZoSTTMR5J+CISiSvZP6eqC92HD4lIP/fj/YCA+v2rZ48ovjSpP6+u20+xrfk1nVDf2MR/vLCat7Yc5qczhzNnQjazxmaw8gfTGZWZRHavHkGT7IEzpQ58dYcPcOmQFKLCw2x5Zid5YpWOAH8Ftqjqb1o89Apwm/vz24DF3b2Wr335koFEhIfx+Nu7nA7FBIiGxia+MX8tSzYd4kfXDeOWiwac9fj1o9PZUHaMXeUnnAnQCwpLKoiPjmBo30SfXbNHdASTBve2Mgud5Ik7/MnALcA0EVnr/rgaeAS4QkR2ANPdXweU1MQYPp+fxcI1+4JqRYXxjsYm5VsL1vH6hgP84Oo87pg88FPnXDc6HRF4ZW3wVGYtLK5krJcKprVnel4qe46cDKofnt7miVU676uqqOooVR3j/vinqh5V1QJVHaKq01W1whMB+9pXLh+EKjzx7m6nQzF+rLFJeeCldbyybj/fvTKXuy8b1Op5aYkxXDy4N4vXlgVFzabq066Cab4czmk2bWgqYLtuO8N22p5HZs84Pjs2gxdW7qX8eK3T4Rg/1NSkzP37ehauKeNbV1zA16bktHv+zNEZFB89xfp9x3wUofes2VvlKpjm5R22rcnsGcfQvgnW3LwTLOF3wL1TBlPf2MRf39/jdCjGzzQ1KT9YtJGXivbxn9Ny+M+CIed9zowRfYkKD2NxEAzrFBVXECYwJtt7BdPaMz0vjaKSSqpO1Tly/UBjCb8DBqXEc/XIfjz7cYlt5zZnqCo/emUTL6zcy9emDOabV1zQoeclxUYybWgqr67fT2OAN9wpLKkkr18i8V4smNaegrxU165b63XbIZbwO+i+qTmcqG3gqQ+LnQ7F+AFV5b9f28wzH5dwz2WDeGBGLq4Fax0zc0w65cdr+WhX4Bbpa3AXTHNi/L7Z6Mxk+sRHWzG1DrKE30F5/RKZnpfKkx/u4WRtg9PhGAepKg//aytPflDMHZMH8OBVQzuV7AGmDk0lITqCRWsDt2LmlgPHOVXnm4JpbQkLE6YNTeGd7eW267YDLOF3wn1Tc6g6Vc9zK0qcDsU4RFWZt2QbT7y7m1sv6s9/XTus08keXEX6rhzRl39vPMjpcyppBorCEtfCu/wBvp+wbakgL43jpxtYtScgFwL6lCX8Thib3ZPJOb3583t7Avab1HTPb9/awWNv72LOhGx+fN3wLiX7ZjPHZHCitoHlAbp5qLCkkn5JMWQ4XATu0iF9iIoIs9U6HWAJv5Pum5pD+fFaXiosdToU42P/u3QH/7N0BzfnZ/LQrBGEdXOj0UWDe5OSEB2QwzqqSlFxpaPDOc3ioiK4eHBvlm49FBR7G7zJEn4nXTSoN+Oyk/njO7ttzDCEPP72Ln795nZuGJvBwzeM6nayB1fVx+tGpbN8aznHagJr9VdZVQ0Hq0+T7wcJH1zDOiVHT9mu2/OwhN9JIsLXp+VQVlUTVC3qTNv+8t5ufvHvrVw/Op15N432aAmBmWPSqWtsYsnGgx57TV9oLpjm9Ph9swL3rlsb1mmfJfwumJqbyrB+iTz+9q6AX0dt2vfkB3v42etbuGZkP35zs2eTPcCozCQG9ukRcMM6RSWVxEWFM7RvgtOhAK5mMsP6JVqZhfOwhN8FIsJ9U3PYfeQk/9p44PxPMAHpmY+K+cmrm5kxPI3fzR5DRLjnv11EhOtHp/PR7qMcqj7t8df3luaCad54T7pqel4qRSWVVJ60Xbdt8Z9/rQBz5Yi+DErpwR+W77KJoiD0/Iq9/HDxJqbnpfK/c8YR6cXENnNMOqrw6rrAKLVworaBrQerGe9A/Zz2FOSl0aSwfJsN67TFEn4XhYcJX5uSw5YD1SwL0GV1pnULCkv5/j82MDU3hT98cVybPWk9ZVBKPKMykwKmts6avZU0KX4zYdtsZEYSKQnR1uu2HZbwu2HmmHQye8by6PKddpcfJBau3sf3/r6eS4f04fEvjSc6Itwn1w2kxiiFxZWECYx1qGBaW8LChIKhqbyzvZy6BltB1xpL+N0QGR7GVy4fzJq9VQFdE8W4LF5bxndeWsdFg3rz51vziYn0TbIHV8IXISDu8otKKsntm0hCTKTToXxKQV4aJ2obWGm7bltlCb+bbhqfSWpCNI8ut2bngez19Qf41oJ15A/oxV9u822yB1d3tYsH9+YVP2+M0tDYxJq9lX43nNPskpw+REeEWTG1NniqifnfROSwiGxscayXiLwpIjvcf/rn/5BuiokM5+5LB/HhrqOs3lvpdDimC5ZsOsj989cwNiuZJ2+/kLgoZ0r9BkJjlK0Hj3OyrpH8Af757RwbFc7knD6267YNnrrDfwq48pxjc4GlqjoEWOr+Oih9YWI2sZFhfOGJjxk493UmP7LMNmUFiKVbDvH151czMjOJJ++4kB4O1XUHuHJkX6Iiwvx6TX7zhit/KKnQloK8VEorathx2P/nQ3zNIwlfVd8Fzh00mwk87f78aWCWJ67lj97cfIj6RuV0QxOKa9v5gws3WNL3c29vO8y9z64mr18iT985wfEx6cSYSKblpvLqugN+u6GvsKSSvonOF0xrT8HQNAAb1mmFN8fw01S1eVfSQSDNi9dy1Lwl22g45xu0pr6ReUu2ORSROZ/3dpRzzzNFDEmL55k7J5LoJxOQM8ekc+RELR/uOuJ0KK0qKq5g/ICe3aoS6m19k2IYkZFoyzNb4ZNJW3UNprV6yyIi94hIoYgUlpcHZpuy/VU1nTpunPXhriN8+elCBvXpwbN3TSQpzj+SPXzSGMUfV+vsr6ph/zH/KZjWnoKhaazeW8nRE7VOh+JXvJnwD4lIPwD3n63+uFXVJ1Q1X1XzU1JSvBiO96S38ettW8eNc1bsPspdTxXSv3ccz315Ij17RDkd0ln8uTFKYXPBND/bYdua6XlpqMJy63V7Fm8m/FeA29yf3wYs9uK1HPXAjFxiW1nG96VJ2Q5EY9pSVFLBHU+tIj05hue+PIne8dFOh9SqWWNdjVH8bQd3UXEFcVHh5PXzj4Jp7RmRkUhaYrQVUzuHR5YkiMgLwBSgj4jsA34EPAIsEJG7gBLgZk9cyx/NGpsBuMby91fVkJoYzanaBp75qIQbxmWSlhjjcISha9GasjP/LgC946N44e5JpCT4Z7IHmDTI1Rhl8doyrh7Zz+lwzigsqWRMln8VTGuLiDBtaBqvrC2jtqHRZzum/Z1HEr6qzmnjoQJPvH4gmDU240ziB9hYdozP/+kj7nxqFQu+cpGjy/1C1aI1ZTy4cAM1LYZGTpxu4MNdR8/6t/I3zY1Rnv24hGM19STFOj/HcKK2gS0Hqvn61BynQ+mw6XmpvLByLyt2V3DZBYE5XOxp/v+jOkCNyEji0S+OY+vB43z9+dU0WHcsn5u3ZNtZyR7gdENTQKyemjXW1Rjl335Sfnvt3iqaFMb7ScOTjpic04eYyDAb1mnBEr4XTc1N5aczR7B8Wzn/9com2/nnY4G8empkhqsxir+s1iksqUD8sGBae2Iiw7kkpw9vbTls33tulvC97AsTs7l3ymCeX7GXP76z2+lwQkpiG0MhgbB6qmVjlIPHnG+MUlRSSW5agt/sV+iogrw0yqpq2HbouNOh+AVL+D7wwGdyuW50Or/491ZeCZAmF4FuV/kJTtbWc25HwtjIcB6YketMUJ3U3BjltfXO/p9pbFLW7K3y2/o57WnudWubsFws4ftAWJjwq5tGMWFAL76zYJ2VbvWyhsYmvrVgHfExkfz4umFkJMciQEZyLA/fMNKvJ2xb8pfGKFsPVnOitiEg1t+fKzUxhlGZSVZmwc0Svo9ER4TzxK3jyewVy93/VxgQjS4C1eNv72JdaRU/mzWCWy8eyAdzp7HnkWv4YO60gEn2zWaOyXC8MUogFExrT8HQNNaWVnHEdt1awvel5Lgonrp9AhFhwu1PrrT/gF6wsewYv1+6g+tGp3PtqHSnw+m260b1c7wxSmFxJakJ0WT29P+5j9YU5KWiit9tZHOCJXwfy+4dx19vv5Dy47Xc9XQhNXX+tX0+kJ2ub+RbC9bSq0cUP5053OlwPMIfGqMUlVSS7+cF09ozPD2RfkkxtjwTS/iOGJOVzP/MHsv6fVXcP3+N35bCDTS/eXM72w+d4Jc3jiI5zr9q5HTHzDGuxijrHGiMcuBYDWVVNYwPwPH7Zq5dt6m8t+OI39Un8jVL+A75zPC+/Ne1w3hj8yF+9vpmp8MJeCt2H+XP7+3mixOzmZKb6nQ4HnXlCFdjlMUONEYpLG4umBaY4/fNpuelcaqukY93h3bvaUv4Drpj8kDunDyQJz8o5m/v73E6nIB1oraBb7+0juxecXz/6jynw/G4lo1RfL1ju6ikktjIcIalJ/r0up520eDexEaGh/zyTEv4DvvBNXnMGJ7GT1/fzL83HnQ6nID0s9c2s7+qhl/fNDpoaxbNGutqjPKRj+9Qi0oqGZ2VRGQAFExrT0xkOJcM6cPSLaHd6zaw/xWDQHiY8LvPj2V0ZjL3z1/DGmuE3inLth5i/qpS7rlsMPkBVOels6bk+r4xysnaBjYfqA7I9fetmZ6Xyv5jp9lyIHR33VrC9wOxUeH85bZ80hJj+PLThZQcPel0SAGh4mQd3315A0P7JvDNK4Y4HY5XOdEYZV1pFY1NyvgA3GHbmqlndt2G7modS/h+ok98NE/dcSGNqtzx5CoqT9Y5HZJfU1X+36INHKup4zc3jwmJeue+boxSWFKJCIzLDo6En5oQw+isZN4K4fX4lvD9yKCUeP58az77qmq455nCkF9C1p5X1u3nnxsO8s0rLgj4CcWOmjSoN6nuxii+UFhSyQWpCX5Rj99Tpg9NZV1pFYePO1+QzgmW8P3MhQN68eubRrOquJLvvLSOJluj/ykHjtXww0UbGd+/J1+5bLDT4fhMeJhw3eh0lm8t59ipeq9eq7FJWVNSGTTDOc3Bfz1BAAAW5UlEQVQK8tIAWB6id/mW8P3QdaPTmXvVUF5bf4BfBkCzDl9SVb778nrqG5Vf3zSa8HPLYQa5mWPcjVE2ebcxyvZDxzle2xDw6+/PldcvgfSkGN4K0eWZXk/4InKliGwTkZ0iMtfb1wsWX7lsEF+cmM0f39nFcytKnA7Hbzy7Yi/v7TjC96/JY0CfHk6H43PNjVEWrfHuap3CkuYNV8GxQqeZiFCQl8b7Ibrr1qsJX0TCgT8AVwHDgDkiMsyb1wwWIsJPrh/OtKGp/HDRxpD9FbSlPUdO8vPXt3DZBSl8aWK20+E4QkSYOSadj/d4tzFKUXEFKQnRZPUKzIJp7SnIS6WmvpGPdoXerltv3+FPAHaq6m5VrQPmAzO9fM2gEREexv/OGcuw9ETue341G8t8X0vFXzQ0NvHtBWuJDBd++blRAVvIyxOuH+39xiiFJZXk9w/cgmntmTSoN3FR4SFZI9/bCT8DKG3x9T73sTNE5B4RKRSRwvLyci+HE3h6REfwt9supGdcFHc8tYqyAOjH6g1/enc3q/dW8dNZI+ibFON0OI5qboyyyEurdQ5Vn2ZfZU3A1r8/n5jIcC4d0odlW0Ov163jk7aq+oSq5qtqfkpKitPh+KXUxBievONCTtc3cseTKzlW490VGv5m8/5qfvfWdq4Z2Y/rRwd+jXtPmDkmg41l1ew87PnGKGcKpgXxzuWCvDQOHDvNpv3VTofiU95O+GVAVouvM93HTCddkJbAn740nj1HTnLvs0XUNfi2iJZTahtcNe6T46L46awRQTnE0BXXjepHmOCVHsmFJRXERIYxPIj3N0wbmopI6PW69XbCXwUMEZGBIhIFzAZe8fI1g9bFOX145IZRfLjrKHMXrg+JX0d/++YOth48zi8+N5JePYKnxn13uRqj9GGxFxqjFJVUMjozOeALprWnT3w0Y7KSWbo1tMbxvfovqqoNwNeBJcAWYIGqbvLmNYPd58Zn8o3pQ1i4uozfvbXD6XC8alVxBX96dxdzJmQxbWia0+H4nevHpFPi4cYop+oa2LS/mvwg23DVmvSkGNbvO8bAua8z+ZFlLFoT/IMPXv8Rrqr/VNULVHWwqj7k7euFgvsLhnDj+Ex+v3QHLxWWnv8JAehkbQPfXrCOzJ6x/OAaW8nbmubGKJ5MVGvdBdOCbf39uRatKTuz+UqBsqoaHly4IeiTfvD+zhbERISff3Ykk3N68+DCDby/44jTIXncQ//cQmnlKX590xjig7TGfXclxkRSMDSV19Z7rjFKkXvCNlgKprVl3pJt1J4zD1ZT38i8IN/Zbgk/QEVFhPH4l8YzOCWee58tYuvB4FltsHzbYZ5fsZe7Lx3EhIHBfafZXTPHeLYxSmFJJRekxZMUFzwF01qzv43lzW0dDxaW8ANYYkwkT95xIbFR4dz55CoOVQd+BcCqU3V87+X1XJAWz7euuMDpcPzelNxUEmIiPFJqoalJWb23MqAblndUenLrO4jbOh4sLOEHuPTkWP52+4Ucq6nnjidXcaK2wemQuuWHizdRcdJV4z4mMvhr3HdXTGQ4V43oy5JN3W+Msv3wcY6fDr6Caa15YEYusef8/4qNDOOBGbkOReQblvCDwIiMJB794ji2HTrOjY9/yMWPLA3IlQevrNvPq+v2843pQxiRkeR0OAFj5hjPNEb5ZMNV8Cf8WWMzePiGkWS0uKO/ZVJ/Zo3NaOdZgc8SfpCYmpvKDeMy2HrwOPurTgfcyoND1af54aKNjMlK5quXh06Ne09obozS3X/nopJK+sRHk90rzkOR+bdZYzP4YO40tv/sKvrER1F89JTTIXmdJfwg8uHOT0/cBcLKg+Ya97UNjfz65tFEBPGGH29obozy9rbuNUYpLKlgfP/kkNvNHBURxo3js1i69XBQzIO1x76zgkigrjx4fuVe3tlezoNX5TE4Jd7pcAJSc2OUf23sWmOUw9WnKa2oCfr1922ZfWEWjU0atPtamlnCDyJtrTBQ4IeLNlJa4X+/spYcPclDr2/hkpw+3DKpv9PhBKyRGUkM6tODxWu7tlqnueFJsLU07KgBfXpw8eDezF9VGtRtRS3hB5HWVh5ER4QxaWAvXlxVypRfvc3989ew5YB/rNlvbFK+vWAd4WHCL28cRViItSv0JBHh+m40RiksriQ6IowR6aE7WT57Qjb7Kmt4f2fwbWRsZgk/iLRceSBARnIsv/jcKOZ/5SLe+95UvnzJQN7afIirfv8etz+5khW7jzpagO3P7+2msKSSn1w/POjXP/vCzDEZqMKrXaigWbTXVTAtKiJ0U8KM4Wn0jItk/qq9TofiNbZnPcjMGpvR6tKytMQYHrw6j69NyeHZFSX87f09fP6JjxmXncy9U3IoGJrq0zvsLQeq+c0b27lyeF8+G+RL4XxlYJ8ejM5MYvG6Mu6+bFCHn1dT18imsmOdek4wio4I53PjMnnqw2LKj9eSkhDtdEgeF7o/zkNUUlwk903N4YO50/jprBGUn6jl7v8rZMbv3uXlon3Ue6gmS3tqGxr55otrSYyN4KHPWo17T7q+C41R1u2roqFJQ2LD1fnMnpBNQ5Py99X7nA7FKyzhh6iYyHBumdSf5d+ewu9njyE8TPjOS+u4/JfL+dv7ezhV570du79/y1Xj/uEbRtE7Pvjuopx0pjFKJ9ofFjVP2FrCJyc1ngkDejF/5d6g7DdhCT/ERYSHMXNMBv+6/1KevONCsnrF8d+vbebiR5bx2ze3U3GyzqPXKyqp5I/v7OLm/EyuGGY17j3tTGOUdfs7nLAKiyvISY0nOc4azADMmZhF8dFTHitI508s4RvAtcpjam4qL37lIv5+78VcOKAXv1+6g8mPLOMnr27ySPP0U3UNfHvBWvolxfLDa63Gvbc0N0ZZW1p13nObmpSikkobzmnhqhH9SIyJ4IWVwbcm3xK++ZTx/Xvy51vzefObl3H1yH4881EJl/9yOd9asJbth453+XUf/udWio+e4lc3jSYhJrjL7zqpuTFKR9bk7yw/QfXpBhvOaSEmMpwbxmWyZONBj/+G6zRL+KZNQ9IS+PXNo3nnu1O55aL+/GvDQT7z23f58tOFZ8Z9O+qd7eU883EJd10ykIsG9/ZSxAY61xjlk4JpobnDti1zJmRT19jEwiCbvO1WwheRm0Rkk4g0iUj+OY89KCI7RWSbiMzoXpjGSRnJsfzouuF8OHca35g+hMKSCj73+Ifc/KePWL718HnHio+dque7L68jJzU+6MvP+ovmxigf7mp/HLqwpILePaIY0Ds0CqZ1VG7fBMZlJ/N8kE3edvcOfyNwA/Buy4MiMgyYDQwHrgQeExErbh7gevaI4hvTL+DDudP4r2uHsa/iFHc8tYqrfv8ei9eWtXk3+aNXNnL0RB2/tRr3PtPcGOV8wzpFJZWM79/Tlsa2Ys6EbHaXn2RVced+m/Vn3Ur4qrpFVVsrxTgTmK+qtaq6B9gJTOjOtYz/iIuK4M5LBvL2A1P51U2jaWxS7p+/lim/eptnPirmdH0ji9aUMfmRZQyY+zqL1u5nel4qIzNDd9u+r3WkMUr58VpKjp4Kifr3XXHNqH4kREfwwsrg2XnrrTH8DKDlFPc+97FPEZF7RKRQRArLy8u9FI7xBldZ2UyWfOMy/nxrPikJ0fxw8SbG//RNvvPSurNW9ry9vTwg6vIHk+bGKEu3tN4YpaikAiAkWhp2RVxUBLPGZvD6hgNUnQqOydvzJnwReUtENrbyMdMTAajqE6qar6r5KSkpnnhJ42NhYcIVw9JYeO/FvHjPJBqalIZzKg6erm/y+7r8waa5McriNjZhFRZXEhURxoiMRB9HFjhmT8iirqGJfwTJzcp5a+mo6vQuvG4ZkNXi60z3MRPERISJg3pT19D6WL6/1+UPNs2NUZ75qIRjp+pJijt7KWxhSSWjM5OIjrB5lbYMT09idGYS81eWcvvFAwJ+rsNbQzqvALNFJFpEBgJDgJVeupbxM21VvrSKmL43a0xGq41RTtc3smn/MRvO6YDZE7LZdug4q/eefyObv+vusszPisg+4CLgdRFZAqCqm4AFwGbg38B9qtr6zJEJOq3V5Y+NDLclmQ4YkZHYamOUdaVV1DdawbSOuG50Oj2iwoNi8ra7q3T+oaqZqhqtqmmqOqPFYw+p6mBVzVXVf3U/VBMoWqvL//ANI1st22y8q63GKIVWMK3D4qMjuH5MBq+t30/16a73DPYHttPWeMWssRl8MHcaex65hg/mTrNk76DWGqMUlVQyOKUHPXtYwbSOmDMhi9P1TSwO8MlbS/jGBLmWjVGgZcE0G7/vqJEZSQxPT+T5laUBvfPWEr4xIaBlY5Rd5Sc4VlMfsg3Lu0JEmD0hmy0Hqlm/75jT4XSZJXxjQkDLxijN4/c2Yds5M8ekExsZHtA9by3hGxMCmhujLFq7n1XFroJpA/v0cDqsgJIYE8m1o/qxeO1+TtR6ryOcN1nCNyZEZPaKZW/FKRauLuNkbUOH6uWbs82ZmM2pusazJsADiSV8Y0LAojVlZ9UyOt3QxIMLN1h9o04am5VMblpCwK7Jt4RvTAiYt2Qbp+vPLnlRU99o9Y06SUSYMyGL9fuOsbEs8CZvLeEbEwLaqmNk9Y0677NjM4mOCAvIyVtL+MaEAKtv5DlJcZFcM7Ifi9bs51RdYE3eWsI3JgRYfSPPmjMxmxO1Dby2/sD5T/YjlvCNCQFW38iz8vv3JCc1PuAmb89bD98YExxmjc2wBO8hIsLsC7P42etb2HqwmqF9A6OJjN3hG2NMF9wwLpOo8DDmryw9/8l+whK+McZ0Qa8eUVw5oi8LV+9rs1G8v7GEb4wxXTR7QhbVpxv454bAmLy1hG+MMV100aDeDOgdFzCTt91tcThPRLaKyHoR+YeIJLd47EER2Ski20RkRnuvY4wxgai5bPKq4kp2Hj7udDjn1d07/DeBEao6CtgOPAggIsOA2cBw4ErgMREJb/NVjDEmQN04PpPIcOGFAJi87W5P2zdUtXmr2cdApvvzmcB8Va1V1T3ATmBCd65ljDH+qE98NJ8ZFhiTt54cw78TaG5WngG0/HG3z33MGGOCzuwJWVSeqmfJpoNOh9Ku8yZ8EXlLRDa28jGzxTk/ABqA5zobgIjcIyKFIlJYXl7e2acbY4zjJg/uQ1avWL9fk3/enbaqOr29x0XkduBaoEA/6e5bBmS1OC3Tfay1138CeAIgPz8/cLsDG2NCVliYMPvCbOYt2caeIyf9tptYd1fpXAl8F7heVU+1eOgVYLaIRIvIQGAIsLI71zLGGH920/hMwsPEr8smd3cM/1EgAXhTRNaKyB8BVHUTsADYDPwbuE9V/Xs2wxhjuiE1MYaCoam8XLiPuoam8z/BAd0qnqaqOe089hDwUHde3xhjAsmcidm8sfkQb24+xDWj+jkdzqfYTltjjPGQy4akkJEc67fDOpbwjTHGQ8LDhJvzs3hvxxH2Hj11/if4mCV8Y4zxoJsvzCRM4MVC/7vLt4RvjDEe1C8plqm5qSwo3Ed9o39N3lrCN8YYD5szIZvy47Us23rY6VDOYgnfGGM8bEpuCmmJ0X5XNtkSvjHGeFhEeBifz8/ine3llFXVOB3OGZbwjTHGC26+0FVd5sVV/lNfxxK+McZ4QWbPOC4bksKCVaU0+MnkrSV8Y4zxkjkTsjhYfZp3tvtHJWBL+MYY4yUFeWn0ifefyVtL+MYY4yWR4WHclJ/Jsq2HOXjstNPhWMI3xhhvmn1hFk0KCwqdn7y1hG+MMV7Uv3cPLsnpw4urSmlscrbHkyV8Y4zxstkTsiirquG9Hc5O3lrCN8YYL/vMsL707hHleM9bS/jGGONlURFhfG58Jm9tOcThaucmby3hG2OMD8y+MIuGJuWlon2OxdDdJuY/FZH17n62b4hIuvu4iMj/iMhO9+PjPBOuMcYEpkEp8Uwc2IsXV5XS5NDkbXfv8Oep6ihVHQO8BvyX+/hVwBD3xz3A4928jjHGBLwvTMxmb8UpPtx11JHrdyvhq2p1iy97AM0/tmYC/6cuHwPJIuJ/HX2NMcaHZgzvS3JcJC841PO222P4IvKQiJQCX+STO/wMoOV09D73sdaef4+IFIpIYXm5f9SbMMYYb4iJDOeGsZm8sekgR0/U+vz65034IvKWiGxs5WMmgKr+QFWzgOeAr3c2AFV9QlXzVTU/JSWl838DY4wJIHMmZFHfqPx9te8nbyPOd4KqTu/gaz0H/BP4EVAGZLV4LNN9zBhjQtqQtATy+/dk/spS7r50ECLis2t3d5XOkBZfzgS2uj9/BbjVvVpnEnBMVQ9051rGGBMs5kzIZveRk6zYU+HT63Z3DP8R9/DOeuAzwP3u4/8EdgM7gT8DX+vmdYwxJmhcPbIfCTERPi+bfN4hnfao6ufaOK7Afd15bWOMCVaxUeHcMDaDF1aV8uOTdfTsEeWT69pOW2OMccDsCdnUNTSxcI3vpjct4RtjjAPy+iUyJiuZF1buxTUo4n2W8I0xxiFzJmSx8/AJikoqfXI9S/jGGOOQa0elEx0u3PLXlQyc+zqTH1nGIi8O8XRr0tYYY0zXvbn5EA0KjfWNAJRV1fDgwg0AzBrbanGCbrE7fGOMcci8Jds+1fawpr6ReUu2eeV6lvCNMcYh+6tqOnW8uyzhG2OMQ9KTYzt1vLss4RtjjEMemJFLbGT4WcdiI8N5YEauV65nk7bGGOOQ5onZeUu2sb+qhvTkWB6YkeuVCVuwhG+MMY6aNTbDawn+XDakY4wxIcISvjHGhAhL+MYYEyIs4RtjTIiwhG+MMSFCfFWWsyNEpBwo6eLT+wBHPBhOoLP342z2fnzC3ouzBcP70V9VU853kl8l/O4QkUJVzXc6Dn9h78fZ7P34hL0XZwul98OGdIwxJkRYwjfGmBARTAn/CacD8DP2fpzN3o9P2HtxtpB5P4JmDN8YY0z7gukO3xhjTDss4RtjTIgIioQvIleKyDYR2Skic52Ox0kikiUiy0Vks4hsEpH7nY7JaSISLiJrROQ1p2Nxmogki8jLIrJVRLaIyEVOx+QUEfmm+3tko4i8ICIxTsfkbQGf8EUkHPgDcBUwDJgjIsOcjcpRDcC3VXUYMAm4L8TfD4D7gS1OB+Enfg/8W1WHAqMJ0fdFRDKA/wTyVXUEEA7MdjYq7wv4hA9MAHaq6m5VrQPmAzMdjskxqnpAVVe7Pz+O6xvaN8W2/ZCIZALXAH9xOhaniUgScBnwVwBVrVPVKmejclQEECsiEUAcsN/heLwuGBJ+BlDa4ut9hHCCa0lEBgBjgRXORuKo3wHfBZqcDsQPDATKgSfdQ1x/EZEeTgflBFUtA34F7AUOAMdU9Q1no/K+YEj4phUiEg/8HfiGqlY7HY8TRORa4LCqFjkdi5+IAMYBj6vqWOAkEJJzXiLSE9dIwEAgHeghIl9yNirvC4aEXwZktfg6030sZIlIJK5k/5yqLnQ6HgdNBq4XkWJcQ33TRORZZ0Ny1D5gn6o2/8b3Mq4fAKFoOrBHVctVtR5YCFzscExeFwwJfxUwREQGikgUromXVxyOyTEiIrjGaLeo6m+cjsdJqvqgqmaq6gBc/y+WqWrQ38W1RVUPAqUikus+VABsdjAkJ+0FJolInPt7poAQmMAO+CbmqtogIl8HluCaaf+bqm5yOCwnTQZuATaIyFr3se+r6j8djMn4j/8AnnPfHO0G7nA4Hkeo6goReRlYjWtl2xpCoMSClVYwxpgQEQxDOsYYYzrAEr4xxoQIS/jGGBMiLOEbY0yIsIRvjDEhwhK+McaECEv4xhgTIv4/iMSwkpMd/cwAAAAASUVORK5CYII=\n",
      "text/plain": [
       "<Figure size 432x288 with 1 Axes>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "plt.title('Profits over training')\n",
    "plt.plot(profits_global, '-o')"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Double Q-learning with target and estimator Networks"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "This is the algorithm we are employing here:\n",
    "\n",
    "![title](./blog_stuff/dqn_target.png)\n",
    "\n",
    "For more details on implementation read the blog [here](). This uses the selection and evaluation \\[NN\\] functions. "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [],
   "source": [
    "# functions to copy parameters between two networks\n",
    "def copy_parameters(q_network, target_network, sess):\n",
    "    # q_network (source) to target_network (target)\n",
    "    # sess: tensorflow session\n",
    "    \n",
    "    # source\n",
    "    source_params = [t for t in tf.trainable_variables() if t.name.startswith(q_network.scope)]\n",
    "    source_params = sorted(source_params, key = lambda v: v.name)\n",
    "    \n",
    "    # target\n",
    "    target_params = [t for t in tf.trainable_variables() if t.name.startswith(target_network.scope)]\n",
    "    target_params = sorted(target_params, key = lambda v: v.name)\n",
    "    \n",
    "    # do assign operations in loop\n",
    "    for s_v, t_v in zip(source_params, target_params):\n",
    "        op = t_v.assign(s_v)\n",
    "        sess.run(op)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [],
   "source": [
    "class DoubleDQN():\n",
    "    # initialization function\n",
    "    def __init__(self, input_dim, scope, is_eval = False):\n",
    "        # input_dim: state size\n",
    "        # is_eval: is being evaluated\n",
    "        # scope: scope of the model\n",
    "        self.state_size = input_dim\n",
    "        self.action_space = 3 # sell, sit, buy\n",
    "        self.scope = scope # name of scope\n",
    "\n",
    "        # code to load previously trained model --> skip for now\n",
    "        # self.model = load_model(\"models/\" + model_name) if is_eval else self._model()\n",
    "        self._build_model()\n",
    "        self._build_loss()\n",
    "\n",
    "    # inhouse functions\n",
    "    def _build_model(self):\n",
    "        self.input_placeholder = tf.placeholder(tf.float32, [None, self.state_size], name = 'inputs')\n",
    "        self.q_placeholder = tf.placeholder(tf.float32, [None, self.action_space], name = 'q_value')\n",
    "\n",
    "        # layers\n",
    "        h1 = tf.contrib.layers.fully_connected(self.input_placeholder, 64)\n",
    "        h2 = tf.contrib.layers.fully_connected(h1, 32)\n",
    "        h3 = tf.contrib.layers.fully_connected(h2, 8)\n",
    "        self.action_pred = tf.contrib.layers.fully_connected(h3, self.action_space, activation_fn = tf.nn.softmax)\n",
    "\n",
    "    def _build_loss(self):\n",
    "        self.loss = tf.reduce_mean(tf.square(self.q_placeholder - self.action_pred))\n",
    "        self.update_step = tf.train.AdamOptimizer(0.001).minimize(self.loss)\n",
    "        \n",
    "    # user callable functions\n",
    "    def save_model(self, frozen = False):\n",
    "        # code to save the model as frozen inference graph (optional) or \n",
    "        # checkpoint file (default)\n",
    "        if frozen:\n",
    "            # convert to frozen graph here\n",
    "            model_path = 'some_path'\n",
    "            print('Model saved at {0}'.format(some_path))\n",
    "            return\n",
    "        return\n",
    "    \n",
    "    # no need for operation functions since everything is called from outside"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [],
   "source": [
    "def train_dqn(q_network,\n",
    "              target_network,\n",
    "              sess,\n",
    "              data,\n",
    "              max_mem_size = 1000,\n",
    "              num_episodes = 50,\n",
    "              train_target_every = 2,\n",
    "              gamma = 0.99,\n",
    "              epsilon_start = 0.99,\n",
    "              epsilon_end = 0.001,\n",
    "              epsilon_decay = 0.995):\n",
    "    # function variables\n",
    "    train_global_step = 0 # global step needed in parameter update\n",
    "    train_loss = [] # training loss in each episode\n",
    "    train_profits = [] # for profits in each episode\n",
    "    \n",
    "    # memory_buffer\n",
    "    memory_buffer = []\n",
    "    \n",
    "    # initialize variables\n",
    "    sess.run(tf.global_variables_initializer())\n",
    "    \n",
    "    # init stuff\n",
    "    epsilon = epsilon_start\n",
    "    \n",
    "    # iterate over each episode\n",
    "    for ep in range(num_episodes):\n",
    "        # for each training episode\n",
    "        state = get_state(data, 0, window_size + 1)\n",
    "        \n",
    "        # init values for new episode\n",
    "        total_profit = 0.0 # total profit in this episode\n",
    "        # q_network.inventory = [] # holdings by q_network\n",
    "        inventory = [] # inventory for this episode\n",
    "        ep_loss = [] # total loss in this episode\n",
    "        \n",
    "        for t in tqdm(range(len_data)):\n",
    "            # take action according to epsilon greedy policy\n",
    "            if np.random.random() > epsilon:\n",
    "                action = np.random.randint(q_network.action_space)\n",
    "            else:\n",
    "                feed_dict = {q_network.input_placeholder: state}\n",
    "                action = sess.run(q_network.action_pred, feed_dict = feed_dict)\n",
    "                action = np.argmax(action[0])\n",
    "            \n",
    "            # next state\n",
    "            next_state = get_state(data, t + 1, window_size + 1)\n",
    "            reward = 0\n",
    "            \n",
    "            # now go according to the actions\n",
    "            if action == 2:\n",
    "                # buy\n",
    "                inventory.append(data[t])\n",
    "                if LOG:\n",
    "                    print('Buy:' + format_price(data[t]))\n",
    "                    \n",
    "            elif action == 0 and len(inventory) > 0:\n",
    "                bought_price = inventory.pop(0) # remove the first element and return the value\n",
    "                profit = data[t] - bought_price # profit this transaction\n",
    "                reward = max(data[t] - bought_price, 0) # reward\n",
    "                total_profit += profit # add to total profit\n",
    "                if LOG:\n",
    "                    print(\"Sell: \" + format_price(data[t]) + \" | Profit: \" + format_price(profit))\n",
    "                    \n",
    "            # condition for done\n",
    "            done = t == len_data - 1\n",
    "            \n",
    "            # add to memory and make sure it's of fixed size\n",
    "            memory_buffer.append((state, action, reward, next_state, done))\n",
    "            if len(memory_buffer) > max_mem_size:\n",
    "                memory_buffer.pop(0)\n",
    "            \n",
    "            # update state\n",
    "            state = next_state\n",
    "            \n",
    "            # train the model\n",
    "            if len(memory_buffer) > batch_size:\n",
    "                \n",
    "                # sample minibatches here\n",
    "                mini_batch = memory_buffer[-batch_size:]\n",
    "                    \n",
    "                # calculate q_value and target_values\n",
    "                for state_t, action_t, reward_t, next_state_t, done_t in mini_batch:\n",
    "                    # condition for calculating y_j\n",
    "                    if done_t:\n",
    "                        target_pred = reward\n",
    "                    \n",
    "                    else:\n",
    "                        feed_dict = {target_network.input_placeholder: next_state_t}\n",
    "                        target_pred = sess.run(target_network.action_pred, feed_dict = feed_dict)\n",
    "                        target_value = reward_t + gamma*np.amax(target_pred[0])\n",
    "                    \n",
    "                    # q_value\n",
    "                    feed_dict = {q_network.input_placeholder: state_t}\n",
    "                    q_values = sess.run(q_network.action_pred, feed_dict = feed_dict)\n",
    "                    q_values[0][action_t] = target_value\n",
    "                    \n",
    "                    # drop epsilon value after every action taken\n",
    "                    if epsilon > epsilon_end:\n",
    "                        epsilon *= epsilon_decay\n",
    "                    \n",
    "                    # update the q_network parameters\n",
    "                    feed_dict = {q_network.input_placeholder: state_t,\n",
    "                                 q_network.q_placeholder: q_values}\n",
    "                    loss, _ = sess.run([q_network.loss, q_network.update_step], feed_dict = feed_dict)\n",
    "                    \n",
    "                    # update the lists\n",
    "                    ep_loss.append(loss)\n",
    "                    \n",
    "                    # update target network\n",
    "                    train_global_step += 1\n",
    "                    if ep % train_target_every == 0:\n",
    "                        copy_parameters(q_network, target_network, sess)\n",
    "                        \n",
    "        # update the outer values\n",
    "        train_loss.append(ep_loss)\n",
    "        train_profits.append(total_profit)\n",
    "        \n",
    "        # print val\n",
    "        print('[*]Episode: {0}, loss: {1}, profits: {2}, epsilon: {3}'\\\n",
    "              .format(ep + 1, np.mean(train_loss[-1]), train_profits[-1], epsilon))\n",
    "    \n",
    "    # return the values\n",
    "    return train_loss, train_profits"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {
    "scrolled": false
   },
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 5360/5360 [05:52<00:00, 15.23it/s]\n",
      "  0%|          | 2/5360 [00:00<08:37, 10.36it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[*]Episode: 1, loss: 2090.693359375, profits: 92243.883383, epsilon: 0.0009954452565571535\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 46%|████▌     | 2454/5360 [02:38<03:08, 15.45it/s]"
     ]
    },
    {
     "ename": "KeyboardInterrupt",
     "evalue": "",
     "output_type": "error",
     "traceback": [
      "\u001b[0;31m---------------------------------------------------------------------------\u001b[0m",
      "\u001b[0;31mKeyboardInterrupt\u001b[0m                         Traceback (most recent call last)",
      "\u001b[0;32m<ipython-input-8-84652a47ccd4>\u001b[0m in \u001b[0;36m<module>\u001b[0;34m()\u001b[0m\n\u001b[1;32m      3\u001b[0m \u001b[0mtarget_network\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mAgent_DQN\u001b[0m\u001b[0;34m(\u001b[0m \u001b[0mwindow_size\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;34m'target_network'\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m      4\u001b[0m \u001b[0msess\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mtf\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mSession\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m----> 5\u001b[0;31m \u001b[0mloss\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mprofits\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mtrain_dqn\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mq_network\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mtarget_network\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0msess\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mdata\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m",
      "\u001b[0;32m<ipython-input-7-d3bdbf02cd1d>\u001b[0m in \u001b[0;36mtrain_dqn\u001b[0;34m(q_network, target_network, sess, data, max_mem_size, num_episodes, train_target_every, gamma, epsilon_start, epsilon_end, epsilon_decay)\u001b[0m\n\u001b[1;32m    103\u001b[0m                     feed_dict = {q_network.input_placeholder: state_t,\n\u001b[1;32m    104\u001b[0m                                  q_network.q_placeholder: q_values}\n\u001b[0;32m--> 105\u001b[0;31m                     \u001b[0mloss\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0m_\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0msess\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mrun\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0mq_network\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mloss\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mq_network\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mupdate_step\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mfeed_dict\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mfeed_dict\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m    106\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    107\u001b[0m                     \u001b[0;31m# update the lists\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m/usr/local/lib/python3.6/site-packages/tensorflow/python/client/session.py\u001b[0m in \u001b[0;36mrun\u001b[0;34m(self, fetches, feed_dict, options, run_metadata)\u001b[0m\n\u001b[1;32m    898\u001b[0m     \u001b[0;32mtry\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    899\u001b[0m       result = self._run(None, fetches, feed_dict, options_ptr,\n\u001b[0;32m--> 900\u001b[0;31m                          run_metadata_ptr)\n\u001b[0m\u001b[1;32m    901\u001b[0m       \u001b[0;32mif\u001b[0m \u001b[0mrun_metadata\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    902\u001b[0m         \u001b[0mproto_data\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mtf_session\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mTF_GetBuffer\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mrun_metadata_ptr\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m/usr/local/lib/python3.6/site-packages/tensorflow/python/client/session.py\u001b[0m in \u001b[0;36m_run\u001b[0;34m(self, handle, fetches, feed_dict, options, run_metadata)\u001b[0m\n\u001b[1;32m   1133\u001b[0m     \u001b[0;32mif\u001b[0m \u001b[0mfinal_fetches\u001b[0m \u001b[0;32mor\u001b[0m \u001b[0mfinal_targets\u001b[0m \u001b[0;32mor\u001b[0m \u001b[0;34m(\u001b[0m\u001b[0mhandle\u001b[0m \u001b[0;32mand\u001b[0m \u001b[0mfeed_dict_tensor\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m   1134\u001b[0m       results = self._do_run(handle, final_targets, final_fetches,\n\u001b[0;32m-> 1135\u001b[0;31m                              feed_dict_tensor, options, run_metadata)\n\u001b[0m\u001b[1;32m   1136\u001b[0m     \u001b[0;32melse\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m   1137\u001b[0m       \u001b[0mresults\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0;34m[\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m/usr/local/lib/python3.6/site-packages/tensorflow/python/client/session.py\u001b[0m in \u001b[0;36m_do_run\u001b[0;34m(self, handle, target_list, fetch_list, feed_dict, options, run_metadata)\u001b[0m\n\u001b[1;32m   1314\u001b[0m     \u001b[0;32mif\u001b[0m \u001b[0mhandle\u001b[0m \u001b[0;32mis\u001b[0m \u001b[0;32mNone\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m   1315\u001b[0m       return self._do_call(_run_fn, feeds, fetches, targets, options,\n\u001b[0;32m-> 1316\u001b[0;31m                            run_metadata)\n\u001b[0m\u001b[1;32m   1317\u001b[0m     \u001b[0;32melse\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m   1318\u001b[0m       \u001b[0;32mreturn\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_do_call\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0m_prun_fn\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mhandle\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mfeeds\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mfetches\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m/usr/local/lib/python3.6/site-packages/tensorflow/python/client/session.py\u001b[0m in \u001b[0;36m_do_call\u001b[0;34m(self, fn, *args)\u001b[0m\n\u001b[1;32m   1320\u001b[0m   \u001b[0;32mdef\u001b[0m \u001b[0m_do_call\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mfn\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;34m*\u001b[0m\u001b[0margs\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m   1321\u001b[0m     \u001b[0;32mtry\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m-> 1322\u001b[0;31m       \u001b[0;32mreturn\u001b[0m \u001b[0mfn\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m*\u001b[0m\u001b[0margs\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m   1323\u001b[0m     \u001b[0;32mexcept\u001b[0m \u001b[0merrors\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mOpError\u001b[0m \u001b[0;32mas\u001b[0m \u001b[0me\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m   1324\u001b[0m       \u001b[0mmessage\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mcompat\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mas_text\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0me\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mmessage\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m/usr/local/lib/python3.6/site-packages/tensorflow/python/client/session.py\u001b[0m in \u001b[0;36m_run_fn\u001b[0;34m(feed_dict, fetch_list, target_list, options, run_metadata)\u001b[0m\n\u001b[1;32m   1305\u001b[0m       \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_extend_graph\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m   1306\u001b[0m       return self._call_tf_sessionrun(\n\u001b[0;32m-> 1307\u001b[0;31m           options, feed_dict, fetch_list, target_list, run_metadata)\n\u001b[0m\u001b[1;32m   1308\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m   1309\u001b[0m     \u001b[0;32mdef\u001b[0m \u001b[0m_prun_fn\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mhandle\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mfeed_dict\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mfetch_list\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m/usr/local/lib/python3.6/site-packages/tensorflow/python/client/session.py\u001b[0m in \u001b[0;36m_call_tf_sessionrun\u001b[0;34m(self, options, feed_dict, fetch_list, target_list, run_metadata)\u001b[0m\n\u001b[1;32m   1407\u001b[0m       return tf_session.TF_SessionRun_wrapper(\n\u001b[1;32m   1408\u001b[0m           \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_session\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0moptions\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mfeed_dict\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mfetch_list\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mtarget_list\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m-> 1409\u001b[0;31m           run_metadata)\n\u001b[0m\u001b[1;32m   1410\u001b[0m     \u001b[0;32melse\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m   1411\u001b[0m       \u001b[0;32mwith\u001b[0m \u001b[0merrors\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mraise_exception_on_not_ok_status\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m \u001b[0;32mas\u001b[0m \u001b[0mstatus\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;31mKeyboardInterrupt\u001b[0m: "
     ]
    }
   ],
   "source": [
    "# run the model\n",
    "q_network = DoubleDQN(window_size, 'q_network')\n",
    "target_network = DoubleDQN( window_size, 'target_network')\n",
    "sess = tf.Session()\n",
    "loss, profits = train_dqn(q_network, target_network, sess, data)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Dueling DQN (DDQN)\n",
    "\n",
    "Below is the architecture and code for dueling DQN. In this architecture, the network calculates two sepearate values Advantage `A(s,a)` and value `v(s)` for any given state, the q-value is then given by `Q(s,a) = A(s,a) + v(s)`\n",
    "\n",
    "![title](./blog_stuff/dueling_dqn.png)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [],
   "source": [
    "class DDQN():\n",
    "    def __init__(self, input_dim, scope):\n",
    "        self.state_size = input_dim\n",
    "        self.scope = scope\n",
    "        self.action_space = 3\n",
    "        \n",
    "        # placeholders\n",
    "        self.input_placeholder = tf.placeholder(tf.float32, [None, self.state_size], name = 'inputs')\n",
    "        self.q_placeholder = tf.placeholder(tf.float32, [None, self.action_space], name = 'q_value')\n",
    "        \n",
    "        # build model\n",
    "        self._build_model()\n",
    "        self._build_loss()\n",
    "        \n",
    "    def _build_model(self):\n",
    "        # layers\n",
    "        h1 = tf.contrib.layers.fully_connected(self.input_placeholder, 64)\n",
    "        common_h2 = tf.contrib.layers.fully_connected(h1, 32)\n",
    "        \n",
    "        # value network layers\n",
    "        val_h3 = tf.contrib.layers.fully_connected(common_h2, 8)\n",
    "        self.value = tf.contrib.layers.fully_connected(val_h3, 1)\n",
    "        \n",
    "        # advantage network layers\n",
    "        adv_h3 = tf.contrib.layers.fully_connected(common_h2, 16)\n",
    "        self.advantage = tf.contrib.layers.fully_connected(adv_h3, self.action_space)\n",
    "        \n",
    "        # get the final q value\n",
    "        # tensorflow automatically perform the calculation of type [1,1] + [1,3] = [1,3]\n",
    "        # Q(s,a) = V(s) + (A(s,a) - 1/|A|(sum(A(s,a))))\n",
    "        self.action_pred = self.value + (self.advantage - tf.reduce_mean(self.advantage, axis = 1, keepdims = True)) \n",
    "        \n",
    "    def _build_loss(self):\n",
    "        self.loss = tf.reduce_mean(tf.square(self.action_pred - self.q_placeholder))\n",
    "        self.train_step = tf.train.AdamOptimizer().minimize(self.loss)\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [],
   "source": [
    "def train_ddqn(network,\n",
    "               sess,\n",
    "               data,\n",
    "               max_mem_size = 1000,\n",
    "               num_episodes = 50,\n",
    "               gamma = 0.99,\n",
    "               epsilon_start = 0.99,\n",
    "               epsilon_end = 0.001,\n",
    "               epsilon_decay = 0.995):\n",
    "    # function variables\n",
    "    train_global_step = 0 # global step needed in parameter update\n",
    "    train_loss = [] # training loss in each episode\n",
    "    train_profits = [] # for profits in each episode\n",
    "    \n",
    "    # memory_buffer\n",
    "    memory_buffer = []\n",
    "    \n",
    "    # initialize variables\n",
    "    sess.run(tf.global_variables_initializer())\n",
    "    \n",
    "    # init stuff\n",
    "    epsilon = epsilon_start\n",
    "    \n",
    "    # iterate over each episode\n",
    "    for ep in range(num_episodes):\n",
    "        # for each training episode\n",
    "        state = get_state(data, 0, window_size + 1)\n",
    "        \n",
    "        # init values for new episode\n",
    "        total_profit = 0.0 # total profit in this episode\n",
    "        # q_network.inventory = [] # holdings by q_network\n",
    "        inventory = [] # inventory for this episode\n",
    "        ep_loss = [] # total loss in this episode\n",
    "        \n",
    "        for t in tqdm(range(len_data)):\n",
    "            # take action according to epsilon greedy policy\n",
    "            if np.random.random() > epsilon:\n",
    "                action = np.random.randint(q_network.action_space)\n",
    "            else:\n",
    "                feed_dict = {q_network.input_placeholder: state}\n",
    "                action = sess.run(q_network.action_pred, feed_dict = feed_dict)\n",
    "                action = np.argmax(action[0])\n",
    "            \n",
    "            # next state\n",
    "            next_state = get_state(data, t + 1, window_size + 1)\n",
    "            reward = 0\n",
    "            \n",
    "            # now go according to the actions\n",
    "            if action == 2:\n",
    "                # buy\n",
    "                inventory.append(data[t])\n",
    "                if LOG:\n",
    "                    print('Buy:' + format_price(data[t]))\n",
    "                    \n",
    "            elif action == 0 and len(inventory) > 0:\n",
    "                bought_price = inventory.pop(0) # remove the first element and return the value\n",
    "                profit = data[t] - bought_price # profit this transaction\n",
    "                reward = max(data[t] - bought_price, 0) # reward\n",
    "                total_profit += profit # add to total profit\n",
    "                if LOG:\n",
    "                    print(\"Sell: \" + format_price(data[t]) + \" | Profit: \" + format_price(profit))\n",
    "                    \n",
    "            # condition for done\n",
    "            done = t == len_data - 1\n",
    "            \n",
    "            # add to memory and make sure it's of fixed size\n",
    "            memory_buffer.append((state, action, reward, next_state, done))\n",
    "            if len(memory_buffer) > max_mem_size:\n",
    "                memory_buffer.pop(0)\n",
    "            \n",
    "            # update state\n",
    "            state = next_state\n",
    "            \n",
    "            # train the model\n",
    "            if len(memory_buffer) > batch_size:\n",
    "                \n",
    "                # sample minibatches here\n",
    "                mini_batch = memory_buffer[-batch_size:]\n",
    "                    \n",
    "                # calculate q_value and target_values\n",
    "                for state_t, action_t, reward_t, next_state_t, done_t in mini_batch:\n",
    "                    # condition for calculating y_j\n",
    "                    if done_t:\n",
    "                        target_pred = reward\n",
    "                    \n",
    "                    else:\n",
    "                        feed_dict = {network.input_placeholder: next_state_t}\n",
    "                        network_pred = sess.run(network.action_pred, feed_dict = feed_dict)\n",
    "                        target_value = reward_t + gamma*np.amax(network_pred[0])\n",
    "                    \n",
    "                    # q_value\n",
    "                    feed_dict = {network.input_placeholder: state_t}\n",
    "                    q_values = sess.run(network.action_pred, feed_dict = feed_dict)\n",
    "                    q_values[0][action_t] = target_value\n",
    "                    \n",
    "                    # drop epsilon value after every action taken\n",
    "                    if epsilon > epsilon_end:\n",
    "                        epsilon *= epsilon_decay\n",
    "                    \n",
    "                    # update the q_network parameters\n",
    "                    feed_dict = {network.input_placeholder: state_t,\n",
    "                                 network.q_placeholder: q_values}\n",
    "                    loss, _ = sess.run([q_network.loss, q_network.train_step], feed_dict = feed_dict)\n",
    "                    \n",
    "                    # update the lists\n",
    "                    ep_loss.append(loss)\n",
    "                    \n",
    "        # update the outer values\n",
    "        train_loss.append(ep_loss)\n",
    "        train_profits.append(total_profit)\n",
    "        \n",
    "        # print val\n",
    "        print('[*]Episode: {0}, loss: {1}, profits: {2}, epsilon: {3}'\\\n",
    "              .format(ep + 1, np.mean(train_loss[-1]), train_profits[-1], epsilon))\n",
    "    \n",
    "    # return the values\n",
    "    return train_loss, train_profits"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 5360/5360 [06:39<00:00, 13.42it/s]\n",
      "  0%|          | 2/5360 [00:00<05:33, 16.05it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[*]Episode: 1, loss: 1380.010009765625, profits: 78631.64690499997, epsilon: 0.0009954452565571535\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 5360/5360 [05:39<00:00, 15.80it/s]\n",
      "  0%|          | 2/5360 [00:00<05:45, 15.50it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[*]Episode: 2, loss: 486.006103515625, profits: 31529.893303999997, epsilon: 0.0009954452565571535\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 5360/5360 [05:34<00:00, 16.01it/s]\n",
      "  0%|          | 2/5360 [00:00<05:41, 15.67it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[*]Episode: 3, loss: 1633.228759765625, profits: 82862.40621299998, epsilon: 0.0009954452565571535\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 5360/5360 [05:39<00:00, 15.78it/s]\n",
      "  0%|          | 2/5360 [00:00<05:24, 16.49it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[*]Episode: 4, loss: 1822.4794921875, profits: 89303.44369399996, epsilon: 0.0009954452565571535\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 63%|██████▎   | 3383/5360 [09:32<05:34,  5.91it/s]"
     ]
    },
    {
     "ename": "KeyboardInterrupt",
     "evalue": "",
     "output_type": "error",
     "traceback": [
      "\u001b[0;31m---------------------------------------------------------------------------\u001b[0m",
      "\u001b[0;31mKeyboardInterrupt\u001b[0m                         Traceback (most recent call last)",
      "\u001b[0;32m<ipython-input-7-1607356e5e54>\u001b[0m in \u001b[0;36m<module>\u001b[0;34m()\u001b[0m\n\u001b[1;32m      2\u001b[0m \u001b[0mq_network\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mDDQN\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mwindow_size\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;34m'q_network'\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m      3\u001b[0m \u001b[0msess\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mtf\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mSession\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m----> 4\u001b[0;31m \u001b[0mloss\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mprofits\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mtrain_ddqn\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mq_network\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0msess\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mdata\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m",
      "\u001b[0;32m<ipython-input-6-ac8c47fcd8ad>\u001b[0m in \u001b[0;36mtrain_ddqn\u001b[0;34m(network, sess, data, max_mem_size, num_episodes, gamma, epsilon_start, epsilon_end, epsilon_decay)\u001b[0m\n\u001b[1;32m     91\u001b[0m                     \u001b[0;31m# q_value\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     92\u001b[0m                     \u001b[0mfeed_dict\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0;34m{\u001b[0m\u001b[0mnetwork\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0minput_placeholder\u001b[0m\u001b[0;34m:\u001b[0m \u001b[0mstate_t\u001b[0m\u001b[0;34m}\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m---> 93\u001b[0;31m                     \u001b[0mq_values\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0msess\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mrun\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mnetwork\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0maction_pred\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mfeed_dict\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mfeed_dict\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m     94\u001b[0m                     \u001b[0mq_values\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0;36m0\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0maction_t\u001b[0m\u001b[0;34m]\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mtarget_value\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     95\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m/usr/local/lib/python3.6/site-packages/tensorflow/python/client/session.py\u001b[0m in \u001b[0;36mrun\u001b[0;34m(self, fetches, feed_dict, options, run_metadata)\u001b[0m\n\u001b[1;32m    898\u001b[0m     \u001b[0;32mtry\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    899\u001b[0m       result = self._run(None, fetches, feed_dict, options_ptr,\n\u001b[0;32m--> 900\u001b[0;31m                          run_metadata_ptr)\n\u001b[0m\u001b[1;32m    901\u001b[0m       \u001b[0;32mif\u001b[0m \u001b[0mrun_metadata\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    902\u001b[0m         \u001b[0mproto_data\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mtf_session\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mTF_GetBuffer\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mrun_metadata_ptr\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m/usr/local/lib/python3.6/site-packages/tensorflow/python/client/session.py\u001b[0m in \u001b[0;36m_run\u001b[0;34m(self, handle, fetches, feed_dict, options, run_metadata)\u001b[0m\n\u001b[1;32m   1133\u001b[0m     \u001b[0;32mif\u001b[0m \u001b[0mfinal_fetches\u001b[0m \u001b[0;32mor\u001b[0m \u001b[0mfinal_targets\u001b[0m \u001b[0;32mor\u001b[0m \u001b[0;34m(\u001b[0m\u001b[0mhandle\u001b[0m \u001b[0;32mand\u001b[0m \u001b[0mfeed_dict_tensor\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m   1134\u001b[0m       results = self._do_run(handle, final_targets, final_fetches,\n\u001b[0;32m-> 1135\u001b[0;31m                              feed_dict_tensor, options, run_metadata)\n\u001b[0m\u001b[1;32m   1136\u001b[0m     \u001b[0;32melse\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m   1137\u001b[0m       \u001b[0mresults\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0;34m[\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m/usr/local/lib/python3.6/site-packages/tensorflow/python/client/session.py\u001b[0m in \u001b[0;36m_do_run\u001b[0;34m(self, handle, target_list, fetch_list, feed_dict, options, run_metadata)\u001b[0m\n\u001b[1;32m   1292\u001b[0m     \u001b[0;32mif\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_created_with_new_api\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m   1293\u001b[0m       \u001b[0;31m# pylint: disable=protected-access\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m-> 1294\u001b[0;31m       \u001b[0mfeeds\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mdict\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mt\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_as_tf_output\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mv\u001b[0m\u001b[0;34m)\u001b[0m \u001b[0;32mfor\u001b[0m \u001b[0mt\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mv\u001b[0m \u001b[0;32min\u001b[0m \u001b[0mfeed_dict\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mitems\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m   1295\u001b[0m       \u001b[0mfetches\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0;34m[\u001b[0m\u001b[0mt\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_as_tf_output\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m \u001b[0;32mfor\u001b[0m \u001b[0mt\u001b[0m \u001b[0;32min\u001b[0m \u001b[0mfetch_list\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m   1296\u001b[0m       \u001b[0mtargets\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0;34m[\u001b[0m\u001b[0mop\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_c_op\u001b[0m \u001b[0;32mfor\u001b[0m \u001b[0mop\u001b[0m \u001b[0;32min\u001b[0m \u001b[0mtarget_list\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m/usr/local/lib/python3.6/site-packages/tensorflow/python/client/session.py\u001b[0m in \u001b[0;36m<genexpr>\u001b[0;34m(.0)\u001b[0m\n\u001b[1;32m   1292\u001b[0m     \u001b[0;32mif\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_created_with_new_api\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m   1293\u001b[0m       \u001b[0;31m# pylint: disable=protected-access\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m-> 1294\u001b[0;31m       \u001b[0mfeeds\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mdict\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mt\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_as_tf_output\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mv\u001b[0m\u001b[0;34m)\u001b[0m \u001b[0;32mfor\u001b[0m \u001b[0mt\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mv\u001b[0m \u001b[0;32min\u001b[0m \u001b[0mfeed_dict\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mitems\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m   1295\u001b[0m       \u001b[0mfetches\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0;34m[\u001b[0m\u001b[0mt\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_as_tf_output\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m \u001b[0;32mfor\u001b[0m \u001b[0mt\u001b[0m \u001b[0;32min\u001b[0m \u001b[0mfetch_list\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m   1296\u001b[0m       \u001b[0mtargets\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0;34m[\u001b[0m\u001b[0mop\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_c_op\u001b[0m \u001b[0;32mfor\u001b[0m \u001b[0mop\u001b[0m \u001b[0;32min\u001b[0m \u001b[0mtarget_list\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m/usr/local/lib/python3.6/site-packages/tensorflow/python/framework/ops.py\u001b[0m in \u001b[0;36m_as_tf_output\u001b[0;34m(self)\u001b[0m\n\u001b[1;32m    596\u001b[0m     \u001b[0;31m# pylint: disable=protected-access\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    597\u001b[0m     \u001b[0;32massert\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mop\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_c_op\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 598\u001b[0;31m     \u001b[0;32mreturn\u001b[0m \u001b[0mc_api_util\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mtf_output\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mop\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_c_op\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mvalue_index\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m    599\u001b[0m     \u001b[0;31m# pylint: enable=protected-access\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    600\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m/usr/local/lib/python3.6/site-packages/tensorflow/python/framework/c_api_util.py\u001b[0m in \u001b[0;36mtf_output\u001b[0;34m(c_op, index)\u001b[0m\n\u001b[1;32m    183\u001b[0m   \"\"\"\n\u001b[1;32m    184\u001b[0m   \u001b[0mret\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mc_api\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mTF_Output\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 185\u001b[0;31m   \u001b[0mret\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0moper\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mc_op\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m    186\u001b[0m   \u001b[0mret\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mindex\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mindex\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    187\u001b[0m   \u001b[0;32mreturn\u001b[0m \u001b[0mret\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;31mKeyboardInterrupt\u001b[0m: "
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      " 63%|██████▎   | 3383/5360 [09:42<05:40,  5.80it/s]"
     ]
    }
   ],
   "source": [
    "# run the model\n",
    "q_network = DDQN(window_size, 'q_network')\n",
    "sess = tf.Session()\n",
    "loss, profits = train_ddqn(q_network, sess, data)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.5"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}
